From 4b4bd7399c1cc5340c7d54077f83b1789978da50 Mon Sep 17 00:00:00 2001 From: Alexander Kozlov Date: Wed, 30 Mar 2022 16:10:03 +0300 Subject: [PATCH 01/39] Fixed conflicts (#11332) --- README.md | 2 +- docs/IE_PLUGIN_DG/QuantizedNetworks.md | 2 +- .../low_precision_transformations/lpt.md | 6 +- .../Deep_Learning_Model_Optimizer_DevGuide.md | 2 +- docs/MO_DG/prepare_model/FP16_Compression.md | 2 +- .../IR_suitable_for_INT8_inference.md | 4 +- docs/OV_Runtime_UG/Int8Inference.md | 10 +- docs/OV_Runtime_UG/supported_plugins/CPU.md | 2 +- docs/OV_Runtime_UG/supported_plugins/GNA.md | 4 +- docs/OV_Runtime_UG/supported_plugins/GPU.md | 4 +- docs/documentation.md | 2 +- docs/install_guides/pypi-openvino-dev.md | 2 +- .../dldt_deployment_optimization_guide.md | 2 +- .../dldt_deployment_optimization_internals.md | 2 +- .../model_optimization_guide.md | 18 +- docs/optimization_guide/nncf_introduction.md | 9 +- tools/pot/README.md | 77 +++----- tools/pot/configs/README.md | 4 +- .../docs/AccuracyAwareQuantizationUsage.md | 182 ++++++++++++++++++ tools/pot/docs/BestPractices.md | 151 +++++++-------- tools/pot/docs/CLI.md | 31 ++- tools/pot/docs/DefaultQuantizationUsage.md | 152 +++++++++++++++ tools/pot/docs/E2eExample.md | 36 +--- tools/pot/docs/Examples.md | 17 ++ tools/pot/docs/FrequentlyAskedQuestions.md | 37 ++-- tools/pot/docs/InstallationGuide.md | 26 ++- tools/pot/docs/Introduction.md | 51 +++++ .../pot/docs/LowPrecisionOptimizationGuide.md | 2 +- tools/pot/docs/ModelRepresentation.md | 2 +- tools/pot/docs/SaturationIssue.md | 2 +- tools/pot/docs/SimplifiedMode.md | 2 +- tools/pot/docs/code/data_loaders.py | 122 ++++++++++++ .../docs/images/default_quantization_flow.png | 3 + tools/pot/docs/pot_docs.xml | 2 +- .../pot/algorithms/quantization/README.md | 28 +-- .../quantization/accuracy_aware/README.md | 57 ++---- .../algorithms/quantization/default/README.md | 60 ++---- .../algorithms/quantization/ranger/README.md | 59 ++---- .../tunable_quantization/README.md | 2 +- tools/pot/openvino/tools/pot/api/README.md | 103 +++------- .../pot/api/samples/3d_segmentation/README.md | 30 ++- .../openvino/tools/pot/api/samples/README.md | 41 ++-- .../pot/api/samples/classification/README.md | 27 ++- .../pot/api/samples/face_detection/README.md | 29 ++- .../api/samples/object_detection/README.md | 29 ++- .../pot/api/samples/segmentation/README.md | 27 ++- .../tools/pot/api/samples/speech/README.md | 27 ++- tools/pot/openvino/tools/pot/version.py | 2 +- 48 files changed, 878 insertions(+), 615 deletions(-) create mode 100644 tools/pot/docs/AccuracyAwareQuantizationUsage.md create mode 100644 tools/pot/docs/DefaultQuantizationUsage.md create mode 100644 tools/pot/docs/Examples.md create mode 100644 tools/pot/docs/Introduction.md create mode 100644 tools/pot/docs/code/data_loaders.py create mode 100644 tools/pot/docs/images/default_quantization_flow.png diff --git a/README.md b/README.md index 79bc1648117..225479c2f13 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Please report questions, issues and suggestions using: [Open Model Zoo]:https://github.com/openvinotoolkit/open_model_zoo [OpenVINO™ Runtime]:https://docs.openvino.ai/latest/openvino_docs_OV_Runtime_User_Guide.html [Model Optimizer]:https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html -[Post-Training Optimization Tool]:https://docs.openvino.ai/latest/pot_README.html +[Post-Training Optimization Tool]:https://docs.openvino.ai/latest/pot_introduction.html [Samples]:https://github.com/openvinotoolkit/openvino/tree/master/samples [tag on StackOverflow]:https://stackoverflow.com/search?q=%23openvino diff --git a/docs/IE_PLUGIN_DG/QuantizedNetworks.md b/docs/IE_PLUGIN_DG/QuantizedNetworks.md index 0c8ad29c234..42d1b68617d 100644 --- a/docs/IE_PLUGIN_DG/QuantizedNetworks.md +++ b/docs/IE_PLUGIN_DG/QuantizedNetworks.md @@ -9,7 +9,7 @@ For more details about low-precision model representation please refer to this [ During the model load each plugin can interpret quantization rules expressed in *FakeQuantize* operations: - Independently based on the definition of *FakeQuantize* operation. - Using a special library of low-precision transformations (LPT) which applies common rules for generic operations, -such as Convolution, Fully-Connected, Eltwise, etc., and translates "fake-quantized" models into the models with low-precision operations. For more information about low-precision flow please refer to the following [document](../OV_Runtime_UG/Int8Inference.md). +such as Convolution, Fully-Connected, Eltwise, etc., and translates "fake-quantized" models into models with low-precision operations. Here we provide only a high-level overview of the interpretation rules of FakeQuantize. At runtime each FakeQuantize can be split into two independent operations: **Quantize** and **Dequantize**. diff --git a/docs/IE_PLUGIN_DG/plugin_transformation_pipeline/low_precision_transformations/lpt.md b/docs/IE_PLUGIN_DG/plugin_transformation_pipeline/low_precision_transformations/lpt.md index 07c21dcaa9d..7a6d6848761 100644 --- a/docs/IE_PLUGIN_DG/plugin_transformation_pipeline/low_precision_transformations/lpt.md +++ b/docs/IE_PLUGIN_DG/plugin_transformation_pipeline/low_precision_transformations/lpt.md @@ -72,11 +72,7 @@ For example, if you would like to infer a model with `Convolution` operation in > There are several supported quantization approaches on activations and on weights. All supported approaches are described in [Quantization approaches](#quantization-approaches) section below. In demonstrated model [FakeQuantize operation quantization](#fakequantize-operation) approach is used. ### Low precision tools -There are two tools to quantize a model: -1. [Post-Training Optimization Toolkit](@ref pot_docs_LowPrecisionOptimizationGuide) (POT) -2. [Neural Network Compression Framework](https://github.com/openvinotoolkit/nncf) (NNCF) - -Additionally, low precision transformations can handle ONNX quantized models. +For more details on how to get a quantized model, refer to [Model Optimization](@ref openvino_docs_model_optimization_guide) document. ## Quantization approaches LPT transformations support two quantization approaches: diff --git a/docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md b/docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md index 52c92f6174b..3fa95005d43 100644 --- a/docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md +++ b/docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md @@ -42,7 +42,7 @@ The IR is a pair of files describing the model: * .bin - Contains the weights and biases binary data. -> **NOTE**: The generated IR can be additionally optimized for inference by [Post-training Optimization tool](../../tools/pot/README.md) +> **NOTE**: The generated IR can be additionally optimized for inference by [Post-training optimization](../../tools/pot/docs/Introduction.md) > that applies post-training quantization methods. > **TIP**: You also can work with the Model Optimizer inside the OpenVINO™ [Deep Learning Workbench](https://docs.openvino.ai/latest/workbench_docs_Workbench_DG_Introduction.html) (DL Workbench). diff --git a/docs/MO_DG/prepare_model/FP16_Compression.md b/docs/MO_DG/prepare_model/FP16_Compression.md index 41c905b7202..76a8ef02f00 100644 --- a/docs/MO_DG/prepare_model/FP16_Compression.md +++ b/docs/MO_DG/prepare_model/FP16_Compression.md @@ -17,4 +17,4 @@ although for the majority of models accuracy degradation is negligible. For deta compressed `FP16` models refer to [Working with devices](../../OV_Runtime_UG/supported_plugins/Device_Plugins.md) page. > **NOTE**: `FP16` compression is sometimes used as initial step for `INT8` quantization, please refer to -> [Post-Training Optimization tool](../../../tools/pot/README.md) for more information about that. +> [Post-training optimization](../../../tools/pot/docs/Introduction.md) for more information about that. diff --git a/docs/MO_DG/prepare_model/convert_model/IR_suitable_for_INT8_inference.md b/docs/MO_DG/prepare_model/convert_model/IR_suitable_for_INT8_inference.md index 9f7ac357e30..99ed90d9379 100644 --- a/docs/MO_DG/prepare_model/convert_model/IR_suitable_for_INT8_inference.md +++ b/docs/MO_DG/prepare_model/convert_model/IR_suitable_for_INT8_inference.md @@ -3,11 +3,11 @@ ## Introduction OpenVINO Runtime CPU and GPU devices can infer models in the low precision. -For details, refer to [Low Precision Inference on the CPU](../../../OV_Runtime_UG/Int8Inference.md). +For details, refer to [Model Optimization Guide](@ref openvino_docs_model_optimization_guide). Intermediate Representation (IR) should be specifically formed to be suitable for low precision inference. Such an IR is called a Low Precision IR and you can generate it in two ways: -- [Quantize regular IR with the Post-Training Optimization tool](@ref pot_README) +- [Quantize regular IR with the Post-Training Optimization tool](@ref pot_introduction) - Use the Model Optimizer for a model pretrained for Low Precision inference: TensorFlow\* pre-TFLite models (`.pb` model file with `FakeQuantize*` operations) and ONNX\* quantized models. Both TensorFlow and ONNX quantized models could be prepared by [Neural Network Compression Framework](https://github.com/openvinotoolkit/nncf/blob/develop/README.md). diff --git a/docs/OV_Runtime_UG/Int8Inference.md b/docs/OV_Runtime_UG/Int8Inference.md index 2ff1180b977..f8fabaf5d5f 100644 --- a/docs/OV_Runtime_UG/Int8Inference.md +++ b/docs/OV_Runtime_UG/Int8Inference.md @@ -1,4 +1,4 @@ -# Low-Precision 8-bit Integer Inference {#openvino_docs_IE_DG_Int8Inference} +# Low-Precision 8-bit Integer Inference ## Disclaimer @@ -14,9 +14,7 @@ Low-precision 8-bit inference is optimized for: ## Introduction -For 8-bit integer computation, a model must be quantized. You can use a quantized model from [OpenVINO™ Toolkit Intel's Pre-Trained Models](@ref omz_models_group_intel) or quantize a model yourself. For quantization, you can use the following: -- [Post-Training Optimization Tool](@ref pot_docs_LowPrecisionOptimizationGuide) delivered with the Intel® Distribution of OpenVINO™ toolkit release package -- [Neural Network Compression Framework](https://www.intel.com/content/www/us/en/artificial-intelligence/posts/openvino-nncf.html) available on GitHub: https://github.com/openvinotoolkit/nncf +For 8-bit integer computation, a model must be quantized. You can use a quantized model from [OpenVINO™ Toolkit Intel's Pre-Trained Models](@ref omz_models_group_intel) or quantize a model yourself. For more details on how to get quantized model please refer to [Model Optimization](@ref openvino_docs_model_optimization_guide) document. The quantization process adds [FakeQuantize](../ops/quantization/FakeQuantize_1.md) layers on activations and weights for most layers. Read more about mathematical computations in the [Uniform Quantization with Fine-Tuning](https://github.com/openvinotoolkit/nncf/blob/develop/docs/compression_algorithms/Quantization.md). @@ -46,10 +44,10 @@ If you infer the model with the OpenVINO™ CPU plugin and collect performance c ## Low-Precision 8-bit Integer Inference Workflow -For 8-bit integer computations, a model must be quantized. Quantized models can be downloaded from [Overview of OpenVINO™ Toolkit Intel's Pre-Trained Models](@ref omz_models_group_intel). If the model is not quantized, you can use the [Post-Training Optimization Tool](@ref pot_README) to quantize the model. The quantization process adds [FakeQuantize](../ops/quantization/FakeQuantize_1.md) layers on activations and weights for most layers. Read more about mathematical computations in the [Uniform Quantization with Fine-Tuning](https://github.com/openvinotoolkit/nncf/blob/develop/docs/compression_algorithms/Quantization.md). +For 8-bit integer computations, a model must be quantized. Quantized models can be downloaded from [Overview of OpenVINO™ Toolkit Intel's Pre-Trained Models](@ref omz_models_group_intel). If the model is not quantized, you can use the [Post-Training Optimization Tool](@ref pot_introduction) to quantize the model. The quantization process adds [FakeQuantize](../ops/quantization/FakeQuantize_1.md) layers on activations and weights for most layers. Read more about mathematical computations in the [Uniform Quantization with Fine-Tuning](https://github.com/openvinotoolkit/nncf/blob/develop/docs/compression_algorithms/Quantization.md). 8-bit inference pipeline includes two stages (also refer to the figure below): -1. *Offline stage*, or *model quantization*. During this stage, [FakeQuantize](../ops/quantization/FakeQuantize_1.md) layers are added before most layers to have quantized tensors before layers in a way that low-precision accuracy drop for 8-bit integer inference satisfies the specified threshold. The output of this stage is a quantized model. Quantized model precision is not changed, quantized tensors are in original precision range (`fp32`). `FakeQuantize` layer has `levels` attribute which defines quants count. Quants count defines precision which is used during inference. For `int8` range `levels` attribute value has to be 255 or 256. To quantize the model, you can use the [Post-Training Optimization Tool](@ref pot_README) delivered with the Intel® Distribution of OpenVINO™ toolkit release package. +1. *Offline stage*, or *model quantization*. During this stage, [FakeQuantize](../ops/quantization/FakeQuantize_1.md) layers are added before most layers to have quantized tensors before layers in a way that low-precision accuracy drop for 8-bit integer inference satisfies the specified threshold. The output of this stage is a quantized model. Quantized model precision is not changed, quantized tensors are in the original precision range (`fp32`). `FakeQuantize` layer has `levels` attribute which defines quants count. Quants count defines precision which is used during inference. For `int8` range `levels` attribute value has to be 255 or 256. To quantize the model, you can use the [Post-Training Optimization Tool](@ref pot_introduction) delivered with the Intel® Distribution of OpenVINO™ toolkit release package. When you pass the quantized IR to the OpenVINO™ plugin, the plugin automatically recognizes it as a quantized model and performs 8-bit inference. Note, if you pass a quantized model to another plugin that does not support 8-bit inference but supports all operations from the model, the model is inferred in precision that this plugin supports. diff --git a/docs/OV_Runtime_UG/supported_plugins/CPU.md b/docs/OV_Runtime_UG/supported_plugins/CPU.md index 0624751953b..40b5e40e89f 100644 --- a/docs/OV_Runtime_UG/supported_plugins/CPU.md +++ b/docs/OV_Runtime_UG/supported_plugins/CPU.md @@ -47,7 +47,7 @@ CPU plugin supports the following data types as inference precision of internal Selected precision of each primitive depends on the operation precision in IR, quantization primitives, and available hardware capabilities. u1/u8/i8 data types are used for quantized operations only, i.e. those are not selected automatically for non-quantized operations. -See [low-precision optimization guide](@ref pot_docs_LowPrecisionOptimizationGuide) for more details on how to get quantized model. +See [low-precision optimization guide](@ref openvino_docs_model_optimization_guide) for more details on how to get a quantized model. > **NOTE**: Platforms that do not support Intel® AVX512-VNNI have a known "saturation issue" which in some cases leads to reduced computational accuracy for u8/i8 precision calculations. > See [saturation (overflow) issue section](@ref pot_saturation_issue) to get more information on how to detect such issues and possible workarounds. diff --git a/docs/OV_Runtime_UG/supported_plugins/GNA.md b/docs/OV_Runtime_UG/supported_plugins/GNA.md index edb907578ef..67fa0f4255f 100644 --- a/docs/OV_Runtime_UG/supported_plugins/GNA.md +++ b/docs/OV_Runtime_UG/supported_plugins/GNA.md @@ -90,7 +90,7 @@ can cause the user's request to be executed on CPU, thereby unnecessarily increa Intel® GNA essentially operates in the low-precision mode which represents a mix of 8-bit (`i8`), 16-bit (`i16`), and 32-bit (`i32`) integer computations. -GNA plugin users are encouraged to use the [Post-Training Optimization Tool](@ref pot_README) to get a model with quantization hints based on statistics for the provided dataset. +GNA plugin users are encouraged to use the [Post-Training Optimization Tool](@ref pot_introduction) to get a model with quantization hints based on statistics for the provided dataset. Unlike other plugins supporting low-precision execution, the GNA plugin can calculate quantization factors at the model loading time, so you can run a model without calibration. However, this mode may not provide satisfactory accuracy because the internal quantization algorithm is based on heuristics which may or may not be efficient, depending on the model and dynamic range of input data and this mode is going to be deprecated soon. @@ -101,7 +101,7 @@ GNA plugin supports the following data types as inference precision of internal [Hello Query Device C++ Sample](@ref openvino_inference_engine_samples_hello_query_device_README) can be used to print out supported data types for all detected devices. -[POT API Usage sample for GNA](@ref pot_sample_speech_README) demonstrates how a model can be quantized for GNA using POT API in 2 modes: +[POT API Usage sample for GNA](@ref pot_example_speech_README) demonstrates how a model can be quantized for GNA using POT API in 2 modes: * Accuracy (i16 weights) * Performance (i8 weights) diff --git a/docs/OV_Runtime_UG/supported_plugins/GPU.md b/docs/OV_Runtime_UG/supported_plugins/GPU.md index 510d63510ed..a643d7bef76 100644 --- a/docs/OV_Runtime_UG/supported_plugins/GPU.md +++ b/docs/OV_Runtime_UG/supported_plugins/GPU.md @@ -109,7 +109,7 @@ GPU plugin supports the following data types as inference precision of internal Selected precision of each primitive depends on the operation precision in IR, quantization primitives, and available hardware capabilities. u1/u8/i8 data types are used for quantized operations only, i.e. those are not selected automatically for non-quantized operations. -See [low-precision optimization guide](@ref pot_docs_LowPrecisionOptimizationGuide) for more details on how to get quantized model. +For more details on how to get a quantized model, refer to [Model Optimization](@ref openvino_docs_model_optimization_guide) document. Floating-point precision of a GPU primitive is selected based on operation precision in IR except [compressed f16 IR form](../../MO_DG/prepare_model/FP16_Compression.md) which is executed in f16 precision. @@ -298,7 +298,7 @@ The behavior depends on specific parameters of the operations and hardware confi ## GPU Performance Checklist: Summary Since the OpenVINO relies on the OpenCL™ kernels for the GPU implementation. Thus, many general OpenCL tips apply: -- Prefer `FP16` inference precision over `FP32`, as the Model Optimizer can generate both variants and the `FP32` is default. Also, consider [int8 inference](../Int8Inference.md) +- Prefer `FP16` inference precision over `FP32`, as the Model Optimizer can generate both variants and the `FP32` is default. Also, consider [int8 inference](@ref openvino_docs_model_optimization_guide). - Try to group individual infer jobs by using [automatic batching](../automatic_batching.md) - Consider [caching](../Model_caching_overview.md) to minimize model load time - If your application is simultaneously using the inference on the CPU or otherwise loads the host heavily, make sure that the OpenCL driver threads do not starve. You can use [CPU configuration options](./CPU.md) to limit number of inference threads for the CPU plugin. diff --git a/docs/documentation.md b/docs/documentation.md index 36732f14532..cde5955ec26 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -96,7 +96,7 @@ With the [Model Downloader](@ref omz_tools_downloader) and [Model Optimizer](MO_ The [OpenVINO™ Runtime User Guide](./OV_Runtime_UG/openvino_intro.md) explains the process of creating your own application that runs inference with the OpenVINO™ toolkit. The [API Reference](./api_references.html) defines the OpenVINO Runtime API for Python, C++, and C. The OpenVINO Runtime API is what you'll use to create an OpenVINO™ inference application, use enhanced operations sets and other features. After writing your application, you can use the [Deployment with OpenVINO](./OV_Runtime_UG/deployment/deployment_intro.md) for deploying to target devices. ## Tuning for Performance -The toolkit provides a [Performance Optimization Guide](optimization_guide/dldt_optimization_guide.md) and utilities for squeezing the best performance out of your application, including [Accuracy Checker](@ref omz_tools_accuracy_checker), [Post-Training Optimization Tool](@ref pot_README), and other tools for measuring accuracy, benchmarking performance, and tuning your application. +The toolkit provides a [Performance Optimization Guide](optimization_guide/dldt_optimization_guide.md) and utilities for squeezing the best performance out of your application, including [Accuracy Checker](@ref omz_tools_accuracy_checker), [Post-Training Optimization Tool](@ref pot_introduction), and other tools for measuring accuracy, benchmarking performance, and tuning your application. ## Graphical Web Interface for OpenVINO™ Toolkit You can choose to use the [OpenVINO™ Deep Learning Workbench](@ref workbench_docs_Workbench_DG_Introduction), a web-based tool that guides you through the process of converting, measuring, optimizing, and deploying models. This tool also serves as a low-effort introduction to the toolkit and provides a variety of useful interactive charts for understanding performance. diff --git a/docs/install_guides/pypi-openvino-dev.md b/docs/install_guides/pypi-openvino-dev.md index 251edd42738..60ad1cb8e61 100644 --- a/docs/install_guides/pypi-openvino-dev.md +++ b/docs/install_guides/pypi-openvino-dev.md @@ -11,7 +11,7 @@ OpenVINO™ toolkit is a comprehensive toolkit for quickly developing applicatio | [Model Optimizer](../MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) | `mo` |**Model Optimizer** imports, converts, and optimizes models that were trained in popular frameworks to a format usable by OpenVINO components. 
Supported frameworks include Caffe\*, TensorFlow\*, MXNet\*, PaddlePaddle\*, and ONNX\*. | | [Benchmark Tool](../../tools/benchmark_tool/README.md)| `benchmark_app` | **Benchmark Application** allows you to estimate deep learning inference performance on supported devices for synchronous and asynchronous modes. | | [Accuracy Checker](@ref omz_tools_accuracy_checker) and
[Annotation Converter](@ref omz_tools_accuracy_checker_annotation_converters) | `accuracy_check`
`convert_annotation` |**Accuracy Checker** is a deep learning accuracy validation tool that allows you to collect accuracy metrics against popular datasets. The main advantages of the tool are the flexibility of configuration and a set of supported datasets, preprocessing, postprocessing, and metrics.
**Annotation Converter** is a utility that prepares datasets for evaluation with Accuracy Checker. | -| [Post-Training Optimization Tool](../../tools/pot/README.md)| `pot` |**Post-Training Optimization Tool** allows you to optimize trained models with advanced capabilities, such as quantization and low-precision optimizations, without the need to retrain or fine-tune models. Optimizations are also available through the [API](../../tools/pot/openvino/tools/pot/api/README.md). | +| [Post-Training Optimization Tool](../../tools/pot/docs/pot_introduction.md)| `pot` |**Post-Training Optimization Tool** allows you to optimize trained models with advanced capabilities, such as quantization and low-precision optimizations, without the need to retrain or fine-tune models. | | [Model Downloader and other Open Model Zoo tools](@ref omz_tools_downloader)| `omz_downloader`
`omz_converter`
`omz_quantizer`
`omz_info_dumper`| **Model Downloader** is a tool for getting access to the collection of high-quality and extremely fast pre-trained deep learning [public](@ref omz_models_group_public) and [Intel](@ref omz_models_group_intel)-trained models. These free pre-trained models can be used to speed up the development and production deployment process without training your own models. The tool downloads model files from online sources and, if necessary, patches them to make them more usable with Model Optimizer. A number of additional tools are also provided to automate the process of working with downloaded models:
**Model Converter** is a tool for converting Open Model Zoo models that are stored in an original deep learning framework format into the OpenVINO Intermediate Representation (IR) using Model Optimizer.
**Model Quantizer** is a tool for automatic quantization of full-precision models in the IR format into low-precision versions using the Post-Training Optimization Tool.
**Model Information Dumper** is a helper utility for dumping information about the models to a stable, machine-readable format. The developer package also installs the OpenVINO™ Runtime package as a dependency. diff --git a/docs/optimization_guide/dldt_deployment_optimization_guide.md b/docs/optimization_guide/dldt_deployment_optimization_guide.md index 5ddbbd7637e..27fa00e230c 100644 --- a/docs/optimization_guide/dldt_deployment_optimization_guide.md +++ b/docs/optimization_guide/dldt_deployment_optimization_guide.md @@ -17,7 +17,7 @@ ## Deployment Optimizations Overview {#openvino_docs_deployment_optimization_guide_overview} Runtime or deployment optimizations are focused on tuning of the inference _parameters_ (e.g. optimal number of the requests executed simultaneously) and other means of how a model is _executed_. -As referenced in the parent [performance introduction topic](./dldt_optimization_guide.md), the [dedicated document](./model_optimization_guide.md) covers the **model-level optimizations** like quantization that unlocks the [int8 inference](../OV_Runtime_UG/Int8Inference.md). Model-optimizations are most general and help any scenario and any device (that accelerated the quantized models). The relevant _runtime_ configuration is `ov::hint::inference_precision` allowing the devices to trade the accuracy for the performance (e.g. by allowing the fp16/bf16 execution for the layers that remain in fp32 after quantization of the original fp32 model). +As referenced in the parent [performance introduction topic](./dldt_optimization_guide.md), the [dedicated document](./model_optimization_guide.md) covers the **model-level optimizations** like quantization that unlocks the 8-bit inference. Model-optimizations are most general and help any scenario and any device (that e.g. accelerates the quantized models). The relevant _runtime_ configuration is `ov::hint::inference_precision` allowing the devices to trade the accuracy for the performance (e.g. by allowing the fp16/bf16 execution for the layers that remain in fp32 after quantization of the original fp32 model). Then, possible optimization should start with defining the use-case. For example, whether the target scenario emphasizes throughput over latency like processing millions of samples by overnight jobs in the data centers. In contrast, real-time usages would likely trade off the throughput to deliver the results at minimal latency. Often this is a combined scenario that targets highest possible throughput while maintaining a specific latency threshold. diff --git a/docs/optimization_guide/dldt_deployment_optimization_internals.md b/docs/optimization_guide/dldt_deployment_optimization_internals.md index 5899e85f4e7..a9b8e3da622 100644 --- a/docs/optimization_guide/dldt_deployment_optimization_internals.md +++ b/docs/optimization_guide/dldt_deployment_optimization_internals.md @@ -8,7 +8,7 @@ This provides much better performance for the networks than batching especially Compared with the batching, the parallelism is somewhat transposed (i.e. performed over inputs, with much less synchronization within CNN ops): ![](../img/cpu_streams_explained.png) -Notice that [high-level performance hints](../OV_Runtime_UG/performance_hints.md) allows the implementation to select the optimal number of the streams, _depending on the model compute demands_ and CPU capabilities (including [int8 inference](../OV_Runtime_UG/Int8Inference.md) hardware acceleration, number of cores, etc). +Note that [high-level performance hints](../OV_Runtime_UG/performance_hints.md) allows the implementation to select the optimal number of the streams, _depending on the model compute demands_ and CPU capabilities (including [int8 inference](@ref openvino_docs_model_optimization_guide) hardware acceleration, number of cores, etc). ## Automatic Batching Internals As explained in the section on the [automatic batching](../OV_Runtime_UG/automatic_batching.md), the feature performs on-the-fly grouping of the inference requests to improve device utilization. diff --git a/docs/optimization_guide/model_optimization_guide.md b/docs/optimization_guide/model_optimization_guide.md index 50469ea5acb..efd4e9c8af1 100644 --- a/docs/optimization_guide/model_optimization_guide.md +++ b/docs/optimization_guide/model_optimization_guide.md @@ -6,19 +6,19 @@ :maxdepth: 1 :hidden: - pot_README + pot_introduction docs_nncf_introduction - openvino_docs_IE_DG_Int8Inference + (Experimental) Protecting Model @endsphinxdirective - Model optimization assumes applying transformations to the model and relevant data flow to improve the inference performance. These transformations are basically offline and can require the availability of training and validation data. It includes such methods as quantization, pruning, preprocessing optimization, etc. OpenVINO provides several tools to optimize models at different steps of model development: + Model optimization is an optional offline step of improving final model performance by applying special optimization methods like quantization, pruning, preprocessing optimization, etc. OpenVINO provides several tools to optimize models at different steps of model development: - - **Post-training Optimization tool [(POT)](../../tools/pot/README.md)** is designed to optimize the inference of deep learning models by applying post-training methods that do not require model retraining or fine-tuning, like post-training quantization. +- **Model Optimizer** implements optimization to a model, most of them added by default, but you can configure mean/scale values, batch size, RGB vs BGR input channels, and other parameters to speed up preprocess of a model ([Embedding Preprocessing Computation](../MO_DG/prepare_model/Additional_Optimizations.md)). -- **Neural Network Compression Framework [(NNCF)](./nncf_introduction.md)** provides a suite of advanced algorithms for Neural Networks inference optimization with minimal accuracy drop, for example, quantization, pruning algorithms. +- **Post-training Optimization tool** [(POT)](../../tools/pot/docs/Introduction.md) is designed to optimize the inference of deep learning models by applying post-training methods that do not require model retraining or fine-tuning, for example, post-training 8-bit quantization. -- **Model Optimizer** implements optimization to a model, most of them added by default, but you can configure mean/scale values, batch size, RGB vs BGR input channels, and other parameters to speed-up preprocess of a model ([Embedding Preprocessing Computation](../MO_DG/prepare_model/Additional_Optimizations.md)) +- **Neural Network Compression Framework** [(NNCF)](./nncf_introduction.md) provides a suite of advanced methods for training-time model optimization within the DL framework, such as PyTorch and TensorFlow. It supports methods, like Quantization-aware Training and Filter Pruning. NNCF-optimized models can be inferred with OpenVINO using all the available workflows. ## Detailed workflow: @@ -27,9 +27,13 @@ To understand which development optimization tool you need, refer to the diagram: -POT is the easiest way to get optimized models, and usually takes several minutes depending on the model size and used HW. NNCF can be considered as an alternative or addition when the first one does not give accurate results. +Post-training methods are limited in terms of achievable accuracy and for challenging use cases accuracy might degrade. In this case, training-time optimization with NNCF is an option. + +Once the model is optimized using the aforementioned tools it can be used for inference using the regular OpenVINO inference workflow. No changes to the code are required. ![](../img/WHAT_TO_USE.svg) +If you are not familiar with model optimization methods, we recommend starting from [post-training methods](@ref pot_introduction). + ## See also - [Deployment optimization](./dldt_deployment_optimization_guide.md) \ No newline at end of file diff --git a/docs/optimization_guide/nncf_introduction.md b/docs/optimization_guide/nncf_introduction.md index 6ce2234771b..8cc7591f0f7 100644 --- a/docs/optimization_guide/nncf_introduction.md +++ b/docs/optimization_guide/nncf_introduction.md @@ -1,8 +1,8 @@ # Neural Network Compression Framework {#docs_nncf_introduction} -This document describes the Neural Network Compression Framework (NNCF) which is being developed as a separate project outside of OpenVINO™ but it is highly aligned with OpenVINO™ in terms of the supported optimization features and models. It is open-sourced and available on [GitHub](https://github.com/openvinotoolkit/nncf). +This document describes the Neural Network Compression Framework (NNCF) which is distributed as a separate tool but is highly aligned with OpenVINO™ in terms of the supported optimization features and models. It is open-sourced and available on [GitHub](https://github.com/openvinotoolkit/nncf). ## Introduction - Neural Network Compression Framework (NNCF) is aimed at optimizing Deep Neural Network (DNN) by applying optimization methods, such as quantization, pruning, etc., to the original framework model. It mostly provides in-training optimization capabilities which means that optimization methods require model fine-tuning during and after optimization. The diagram below shows the model optimization workflow using NNCF. + Neural Network Compression Framework (NNCF) is aimed at optimizing Deep Neural Network (DNN) by applying optimization methods, such as quantization, pruning, etc., to the original framework model. It provides in-training optimization capabilities which means that optimization methods require model fine-tuning or even re-training. The diagram below shows the model optimization workflow using NNCF. ![](../img/nncf_workflow.png) ### Features @@ -42,7 +42,6 @@ NNCF provides various examples and tutorials that demonstrate usage of optimizat ### Tutorials - [Quantization-aware training of PyTorch model](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/302-pytorch-quantization-aware-training) - [Quantization-aware training of TensorFlow model](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/305-tensorflow-quantization-aware-training) -- (Experimental) [Post-training quantization of PyTorch model](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/112-pytorch-post-training-quantization-nncf) ### Samples - PyTorch: @@ -58,6 +57,6 @@ NNCF provides various examples and tutorials that demonstrate usage of optimizat ## See also - [Compressed Model Zoo](https://github.com/openvinotoolkit/nncf#nncf-compressed-model-zoo) -- [NNCF in HuggingFace Optimum](https://github.com/dkurt/optimum-openvino) -- [OpenVINO™ Post-training Optimization tool](../../tools/pot/README.md) +- [NNCF in HuggingFace Optimum](https://github.com/openvinotoolkit/openvino_contrib/tree/master/modules/optimum) +- [Post-training optimization](../../tools/pot/docs/Introduction.md) diff --git a/tools/pot/README.md b/tools/pot/README.md index d96f2e250be..ecf30fa8d49 100644 --- a/tools/pot/README.md +++ b/tools/pot/README.md @@ -1,22 +1,4 @@ -# Post-Training Optimization Tool {#pot_README} - -@sphinxdirective - -.. toctree:: - :maxdepth: 1 - :hidden: - - pot_InstallationGuide - pot_docs_LowPrecisionOptimizationGuide - pot_compression_algorithms_quantization_README - Best Practices - Command-line Interface - pot_compression_api_README - pot_configs_README - Deep Neural Network Protection - pot_docs_FrequentlyAskedQuestions - -@endsphinxdirective +# Post-Training Optimization Tool ## Introduction @@ -25,48 +7,35 @@ special methods without model retraining or fine-tuning, for example, post-train require a training dataset or a pipeline. To apply post-training algorithms from the POT, you need: * A floating-point precision model, FP32 or FP16, converted into the OpenVINO™ Intermediate Representation (IR) format and run on CPU with the OpenVINO™. -* A representative calibration dataset representing a use case scenario, for example, 300 images. +* A representative calibration dataset representing a use case scenario, for example, 300 samples. Figure below shows the optimization workflow: ![](docs/images/workflow_simple.png) -### Features +To get started with POT tool refer to the corresponding OpenVINO™ [documentation](https://docs.openvino.ai/latest/openvino_docs_model_optimization_guide.html). -* Two post-training 8-bit quantization algorithms: fast [DefaultQuantization](openvino/tools/pot/algorithms/quantization/default/README.md) and precise [AccuracyAwareQuantization](openvino/tools/pot/algorithms/quantization/accuracy_aware/README.md). -* Compression for different hardware targets such as CPU and GPU. -* Multiple domains: Computer Vision, Natural Language Processing, Recommendation Systems, Speech Recognition. -* [Command-line tool](docs/CLI.md) that provides a simple interface for basic use cases. -* [API](openvino/tools/pot/api/README.md) that helps to apply optimization methods within a custom inference script written with OpenVINO Python* API. -* (Experimental) [Ranger algorithm](@ref pot_ranger_README) for the model protection in safety-critical cases. +## Installation +### From PyPI +POT is distributed as a part of OpenVINO™ Development Tools package. For installation instruction please refer to this [document](https://docs.openvino.ai/latest/openvino_docs_install_guides_install_dev_tools.html). -For benchmarking results collected for the models optimized with the POT tool, see [INT8 vs FP32 Comparison on Select Networks and Platforms](@ref openvino_docs_performance_int8_vs_fp32). +### From GitHub +As prerequisites, you should install [OpenVINO™ Runtime](https://docs.openvino.ai/latest/openvino_docs_install_guides_install_runtime.html) and other dependencies such as [Model Optimizer](https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html) and [Accuracy Checker](https://docs.openvino.ai/latest/omz_tools_accuracy_checker.html). -POT is open-sourced on GitHub as a part of OpenVINO and available at https://github.com/openvinotoolkit/openvino/tools/pot. +To install POT from source: +- Clone OpenVINO repository + ```sh + git clone --recusive https://github.com/openvinotoolkit/openvino.git + ``` +- Navigate to `openvino/tools/pot/` folder +- Install POT package: + ```sh + python3 setup.py install + ``` -Further documentation presumes that you are familiar with basic Deep Learning concepts, such as model inference, dataset preparation, model optimization, as well as with the OpenVINO™ toolkit and its components, such as [Model Optimizer](@ref openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide) and [Accuracy Checker Tool](@ref omz_tools_accuracy_checker). - -## Get started - -### Installation -To install POT, follow the [Installation Guide](docs/InstallationGuide.md). - -### Usage options - -![](docs/images/use_cases.png) - -The POT provides three basic usage options: -* **Command-line interface (CLI)**: - * [**Simplified mode**](@ref pot_docs_simplified_mode): use this option if the model belongs to the **Computer Vision** domain and you have an **unannotated dataset** for optimization. This optimization method does not allow measuring model accuracy and might cause its deviation. - * [**Model Zoo flow**](@ref pot_compression_cli_README): this option is recommended if the model is similar to the model from OpenVINO™ [Model Zoo](https://github.com/openvinotoolkit/open_model_zoo) or there is a valid [Accuracy Checker Tool](@ref omz_tools_accuracy_checker) -configuration file for the model that allows validating model accuracy using [Accuracy Checker Tool](@ref omz_tools_accuracy_checker). -* [**Python\* API**](@ref pot_compression_api_README): this option allows integrating the optimization methods implemented in POT into -a Python* inference script that uses [OpenVINO Python* API](https://docs.openvino.ai/latest/openvino_inference_engine_ie_bridges_python_docs_api_overview.html). +After installation POT is available as a Python library under `openvino.tools.pot.*` and in the command line by the `pot` alias. To verify it, run `pot -h`. -POT is also integrated into [Deep Learning Workbench](@ref workbench_docs_Workbench_DG_Introduction) (DL Workbench), a web-based graphical environment -that enables you to to import, optimize, benchmark, visualize, and compare performance of deep learning models. - -### Examples +## Examples OpenVINO provides several examples to demonstrate the POT optimization workflow: @@ -81,12 +50,10 @@ OpenVINO provides several examples to demonstrate the POT optimization workflow: * [Quantization of 3D segmentation model](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/3d_segmentation) * [Quantization of Face Detection model](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/face_detection) * [Quantization of Object Detection model with controable accuracy](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/object_detection) - * [Speech example for GNA device](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/speech) + * [Quantizatin of speech model for GNA device](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/speech) ## See Also -* [Low Precision Optimization Guide](docs/LowPrecisionOptimizationGuide.md) -* [Post-Training Optimization Best Practices](docs/BestPractices.md) -* [POT Frequently Asked Questions](docs/FrequentlyAskedQuestions.md) +* [Performance Benchmarks](https://docs.openvino.ai/latest/openvino_docs_performance_benchmarks_openvino.html) * [INT8 Quantization by Using Web-Based Interface of the DL Workbench](https://docs.openvino.ai/latest/workbench_docs_Workbench_DG_Int_8_Quantization.html) diff --git a/tools/pot/configs/README.md b/tools/pot/configs/README.md index 7f78beb111f..4a075015e0c 100644 --- a/tools/pot/configs/README.md +++ b/tools/pot/configs/README.md @@ -44,8 +44,7 @@ There are two options to define engine parameters in this mode: ## Compression Parameters -This section defines optimization algorithms and their parameters. For more details about parameters of the concrete optimization algorithm, please refer to the corresponding -[documentation](@ref pot_compression_algorithms_quantization_README). +For more details about parameters of the concrete optimization algorithm, see descriptions of [Default Quantization](@ref pot_compression_algorithms_quantization_default_README) and [Accuracy-aware Quantizatoin](@ref accuracy_aware_README) methods. ## Examples of the Configuration File @@ -57,4 +56,3 @@ For details on how to run the Post-Training Optimization Tool with a sample conf ## See Also * [Optimization with Simplified mode](@ref pot_docs_simplified_mode) -* [POT API](@ref pot_compression_api_README) diff --git a/tools/pot/docs/AccuracyAwareQuantizationUsage.md b/tools/pot/docs/AccuracyAwareQuantizationUsage.md new file mode 100644 index 00000000000..f126e0f8899 --- /dev/null +++ b/tools/pot/docs/AccuracyAwareQuantizationUsage.md @@ -0,0 +1,182 @@ +# Quantizing Model with Accuracy Control{#pot_accuracyaware_usage} + +@sphinxdirective + +.. toctree:: + :maxdepth: 1 + :hidden: + + AccuracyAwareQuantization Method + +@endsphinxdirective + +## Introduction +This document assumes that you already tried [Default Quantization](@ref pot_default_quantization_usage) for the same model. In case when it introduces a significant accuracy degradation, the Accuracy-aware Quantization algorithm can be used to remain accuracy within the pre-defined range. This may cause a +degradation of performance in comparison to [Default Quantization](@ref pot_default_quantization_usage) algorithm because some layers can be reverted back to the original precision. + +> **NOTE**: In case of GNA `target_device`, the Accuracy-aware Quantization algorithm behavior is different. It is searching for the best configuration selecting between INT8 and INT16 precisions for weights of each layer. The algorithm works for the `performance` preset only. For the `accuracy` preset, this algorithm is not helpful since the whole model is already in INT16 precision. + +A script for Accuracy-aware Quantization includes four steps: +1. Prepare data and dataset interface +2. Define accuracy metric +3. Select quantization parameters +4. Define and run quantization process + +## Prepare data and dataset interface +This step is the same as in the case of [Default Quantization](@ref pot_default_quantization_usage). The only difference is that `__getitem__()` method should return `(data, annotation)` or `(data, annotation, metadata)` where `annotation` is required and its format should correspond to the expectations of the `Metric` class. `metadata` is an optional field that can be used to store additional information required for post-processing. + +## Define accuracy metric +To control accuracy during the optimization a `openvino.tools.pot.Metric` interface should be implemented. Each implementation should override the following properties: +- `value` - returns the accuracy metric value for the last model output in a format of `Dict[str, numpy.array]`. +- `avg_value` - returns the average accuracy metric over collected model results in a format of `Dict[str, numpy.array]`. +- `higher_better` should return `True` if a higher value of the metric corresponds to better performance, otherwise, returns `False`. Default implementation returns `True`. + +and methods: +- `update(output, annotation)` - calculates and updates the accuracy metric value using the last model output and annotation. The model output and annotation should be passed in this method. It should also contain the model-specific post-processing in case the model returns the raw output. +- `reset()` - resets collected accuracy metric. +- `get_attributes()` - returns a dictionary of metric attributes: + ``` + {metric_name: {attribute_name: value}} + ``` + Required attributes: + - `direction` - (`higher-better` or `higher-worse`) a string parameter defining whether metric value + should be increased in accuracy-aware algorithms. + - `type` - a string representation of metric type. For example, 'accuracy' or 'mean_iou'. + +Below is an example of the accuracy top-1 metric implementation with POT API: +```python +from openvino.tools.pot import Metric + +class Accuracy(Metric): + + # Required methods + def __init__(self, top_k=1): + super().__init__() + self._top_k = top_k + self._name = 'accuracy@top{}'.format(self._top_k) + self._matches = [] # container of the results + + @property + def value(self): + """ Returns accuracy metric value for all model outputs. """ + return {self._name: self._matches[-1]} + + @property + def avg_value(self): + """ Returns accuracy metric value for all model outputs. """ + return {self._name: np.ravel(self._matches).mean()} + + def update(self, output, target): + """ Updates prediction matches. + :param output: model output + :param target: annotations + """ + if len(output) > 1: + raise Exception('The accuracy metric cannot be calculated ' + 'for a model with multiple outputs') + if isinstance(target, dict): + target = list(target.values()) + predictions = np.argsort(output[0], axis=1)[:, -self._top_k:] + match = [float(t in predictions[i]) for i, t in enumerate(target)] + + self._matches.append(match) + + def reset(self): + """ Resets collected matches """ + self._matches = [] + + def get_attributes(self): + """ + Returns a dictionary of metric attributes {metric_name: {attribute_name: value}}. + Required attributes: 'direction': 'higher-better' or 'higher-worse' + 'type': metric type + """ + return {self._name: {'direction': 'higher-better', + 'type': 'accuracy'}} +``` + +An instance of the `Metric` implementation should be passed to `IEEngine` object responsible for model inference. + +```python +metric = Accuracy() +engine = IEEngine(config=engine_config, data_loader=data_loader, metric=metric) +``` + +## Select quantization parameters +Accuracy-aware Quantization uses the Default Quantization algorithm at the initialization step so that all its parameters are also valid and can be specified. Here, we +describe only Accuracy-aware Quantization required parameters: +- `"maximal_drop"` - maximum accuracy drop which has to be achieved after the quantization. Default value is `0.01` (1%). + +## Run quantization + +The code example below shows a basic quantization workflow with accuracy control. `UserDataLoader()` is a placeholder for the implementation of `DataLoader`. + +```python +from openvino.tools.pot import IEEngine +from openvino.tools.pot load_model, save_model +from openvino.tools.pot import compress_model_weights +from openvino.tools.pot import create_pipeline + +# Model config specifies the model name and paths to model .xml and .bin file +model_config = Dict( + { + "model_name": "model", + "model": path_to_xml, + "weights": path_to_bin, + } +) + +# Engine config +engine_config = Dict({"device": "CPU"}) + +algorithms = [ + { + "name": "AccuracyAwareQuantization", + "params": { + "target_device": "ANY", + "stat_subset_size": 300, + 'maximal_drop': 0.02 + }, + } +] + +# Step 1: implement and create user's data loader +data_loader = UserDataLoader() + +# Step 2: implement and create user's data loader +metric = Accuracy() + +# Step 3: load model +model = load_model(model_config=model_config) + +# Step 4: Initialize the engine for metric calculation and statistics collection. +engine = IEEngine(config=engine_config, data_loader=data_loader, metric=metric) + +# Step 5: Create a pipeline of compression algorithms and run it. +pipeline = create_pipeline(algorithms, engine) +compressed_model = pipeline.run(model=model) + +# Step 6 (Optional): Compress model weights to quantized precision +# in order to reduce the size of the final .bin file. +compress_model_weights(compressed_model) + +# Step 7: Save the compressed model to the desired path. +# Set save_path to the directory where the model should be saved +compressed_model_paths = save_model( + model=compressed_model, + save_path="optimized_model", + model_name="optimized_model", +) + +# Step 8 (Optional): Evaluate the compressed model. Print the results. +metric_results = pipeline.evaluate(compressed_model) +``` + +It is worth noting that now the `evaluate` method that can compute accuracy on demand is also available in the `Pipeline` object. + +In case when Accuracy-aware Quantization does not allow achieving the desired accuracy-performance trade-off, it is recommended to try Quantization-aware Training from [NNCF](@ref docs_nncf_introduction). + +## Examples + + * [Quantization of Object Detection model with control of accuracy](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/object_detection) + diff --git a/tools/pot/docs/BestPractices.md b/tools/pot/docs/BestPractices.md index 5972b66e18e..d99079d2c64 100644 --- a/tools/pot/docs/BestPractices.md +++ b/tools/pot/docs/BestPractices.md @@ -1,10 +1,18 @@ -# Post-Training Optimization Best Practices {#pot_docs_BestPractices} -This document describes the most common insights about model optimization using the Post-training Optimization Tool (POT). The post-training optimization usually is -the fastest and easiest way to get a low-precision model because it does not require model fine-tuning and thus, there is no need in the training dataset, pipeline and availability of -the powerful training hardware. In some cases, it may lead to not satisfactory accuracy drop, especially when optimizing the whole model. -However, it can be still helpful for fast performance evaluation in order to understand the possible speed up - when applying one or another optimization method. Before going into details -we suggest reading the following [POT documentation](../README.md). +# Post-Training Quantization Best Practices {#pot_docs_BestPractices} + +@sphinxdirective + +.. toctree:: + :maxdepth: 1 + :hidden: + + Saturation Issue + +@endsphinxdirective + +## Introduction +The [Default Quantization](@ref pot_default_quantization_usage) of the Post-training Optimization Tool (POT) is +the fastest and easiest way to get a quantized model because it requires only some unannotated representative dataset to be provided in most cases. Thus, it is recommended to use it as a starting point when it comes to model optimization. However, it can lead to significant accuracy deviation in some cases. This document is aimed at providing tips to address this issue. > **NOTE**: POT uses inference on the CPU during model optimization. It means the ability to infer the original > floating-point model is a prerequisite for model optimization. @@ -12,69 +20,53 @@ we suggest reading the following [POT documentation](../README.md). > architecture when optimizing for CPU or VNNI-based CPU when quantizing for a non-CPU device, such as GPU, VPU, or GNA. > It should help to avoid the impact of the [saturation issue](@ref pot_saturation_issue) that occurs on AVX and SSE based CPU devices. -## Get Started with Post-Training Quantization - -Post-training quantization is a basic feature of the POT and it has lots of knobs that can be used to get an accurate -quantized model. However, as a starting point we suggest using the `DefaultQuantization` algorithm with default settings. -In many cases it leads to satisfied accuracy and performance speedup. - -A fragment of the configuration file (`openvino/tools/pot/configs/templates/default_quantization_template.json` in the POT directory) with default settings is shown below: -``` -"compression": { - "target_device": "ANY", // Target device, the specificity of which will be taken into account during optimization. - // The default value "ANY" stands for compatible quantization supported by any HW. - "algorithms": [ - { - "name": "DefaultQuantization", // Optimization algorithm name - "params": { - "preset": "performance", // Preset [performance, mixed] which control the quantization - // mode (symmetric, mixed (weights symmetric and activations asymmetric) - // and fully asymmetric respectively) - - "stat_subset_size": 300 // Size of subset to calculate activations statistics that can be used - // for quantization parameters calculation - } +## Improving accuracy after the Default Quantization +Parameters of the Default Quantization algorithm with basic settings are shown below: +```python +{ + "name": "DefaultQuantization", # Optimization algorithm name + "params": { + "preset": "performance", # Preset [performance, mixed] which controls + # the quantization scheme. For the CPU: + # performance - symmetric quantization of weights and activations + # mixed - symmetric weights and asymmetric activations + "stat_subset_size": 300 # Size of subset to calculate activations statistics that can be used + # for quantization parameters calculation } - ] } ``` -In the case of substantial accuracy degradation after applying the `DefaultQuantization` algorithm there are two alternatives to use: +In the case of substantial accuracy degradation after applying this method there are two alternatives: 1. Hyperparameters tuning 2. AccuracyAwareQuantization algorithm -## Tuning Hyperparameters of the DefaultQuantization -The `DefaultQuantization` algorithm provides multiple hyperparameters which can be used in order to improve accuracy results for the fully-quantized model. -Below is a list of best practices which can be applied to improve accuracy without a substantial performance reduction with respect to default settings: -1. The first option that we recommend is to change is `preset` from `performance` to `mixed`. This enables asymmetric quantization of -activations and can be helpful for NNs with non-ReLU activation functions, e.g. YOLO, EfficientNet, etc. +### Tuning Hyperparameters of the Default Quantization +The Default Quantization algorithm provides multiple hyperparameters which can be used in order to improve accuracy results for the fully-quantized model. +Below is a list of best practices that can be applied to improve accuracy without a substantial performance reduction with respect to default settings: +1. The first recommended option is to change the `preset` from `performance` to `mixed`. This enables asymmetric quantization of +activations and can be helpful for models with non-ReLU activation functions, for example, YOLO, EfficientNet, etc. 2. The next option is `use_fast_bias`. Setting this option to `false` enables a different bias correction method which is more accurate, in general, -and applied after model quantization as a part of the `DefaultQuantization` algorithm. +and applied after model quantization as a part of the Default Quantization algorithm. > **NOTE**: Changing this option can substantially increase quantization time in the POT tool. 3. Another important option is a `range_estimator`. It defines how to calculate the minimum and maximum of quantization range for weights and activations. For example, the following `range_estimator` for activations can improve the accuracy for Faster R-CNN based networks: -``` -"compression": { - "target_device": "ANY", - "algorithms": [ - { - "name": "DefaultQuantization", - "params": { - "preset": "performance", - "stat_subset_size": 300 - +```python +{ + "name": "DefaultQuantization", + "params": { + "preset": "performance", + "stat_subset_size": 300 + - "activations": { - "range_estimator": { - "max": { - "aggregator": "max", - "type": "abs_max" - } - } + "activations": { # defines activation + "range_estimator": { # defines how to estimate statistics + "max": { # right border of the quantizating floating-point range + "aggregator": "max", # use max(x) to aggregate statistics over calibration dataset + "type": "abs_max" # use abs(max(x)) to get per-sample statistics } } } - ] + } } ``` @@ -85,43 +77,30 @@ It is assumed that this dataset should contain a sufficient number of representa However, we empirically found that 300 samples are sufficient to get representative statistics in most cases. 5. The last option is `ignored_scope`. It allows excluding some layers from the quantization process, i.e. their inputs will not be quantized. It may be helpful for some patterns for which it is known in advance that they drop accuracy when executing in low-precision. For example, `DetectionOutput` layer of SSD model expressed as a subgraph should not be quantized to preserve the accuracy of Object Detection models. -One of the sources for the ignored scope can be the AccuracyAware algorithm which can revert layers back to the original precision (see details below). +One of the sources for the ignored scope can be the Accuracy-aware algorithm which can revert layers back to the original precision (see details below). -## AccuracyAwareQuantization -In case when the steps above do not lead to the accurate quantized model you may use the so-called `AccuracyAwareQuantization` algorithm which leads to mixed-precision models. -The whole idea behind that is to revert quantized layers back to floating-point precision based on their contribution to the accuracy drop until the desired accuracy degradation with respect to -the full-precision model is satisfied. +## Accuracy-aware Quantization +In case when the steps above do not lead to the accurate quantized model you may use the so-called [Accuracy-aware Quantization](@ref pot_accuracyaware_usage) algorithm which leads to mixed-precision models. +A fragment of Accuracy-aware Quantization configuration with default settings is shown below below: +```python +{ + "name": "AccuracyAwareQuantization", + "params": { + "preset": "performance", + "stat_subset_size": 300, -A fragment of the configuration file with default settings is shown below (`openvino/tools/pot/configs/templates/accuracy_aware_quantization_template.json`): -``` -"compression": { - "target_device": "ANY", // Target device, the specificity of which will be taken into account during optimization. - // The default value "ANY" stands for compatible quantization supported by any HW. - "algorithms": [ - { - "name": "AccuracyAwareQuantization", // Optimization algorithm name - "params": { - "preset": "performance", // Preset [performance, mixed, accuracy] which control the quantization - // mode (symmetric, mixed (weights symmetric and activations asymmetric) - // and fully asymmetric respectively) - - "stat_subset_size": 300, // Size of subset to calculate activations statistics that can be used - // for quantization parameters calculation - - "maximal_drop": 0.01 // Maximum accuracy drop which has to be achieved after the quantization - } - } - ] + "maximal_drop": 0.01 # Maximum accuracy drop which has to be achieved after the quantization } +} ``` -Since the `AccuracyAwareQuantization` calls the `DefaultQuantization` at the first step it means that all the parameters of the latter one are also valid and can be applied to the -accuracy-aware scenario. +Since the Accuracy-aware Quantization calls the Default Quantization at the first step it means that all the parameters of the latter one are also valid and can be applied to the accuracy-aware scenario. -> **NOTE**: In general case, possible speedup after applying the `AccuracyAwareQuantization` algorithm is less than after the `DefaultQuantization` when the model gets fully-quantized. +> **NOTE**: In general case, possible speedup after applying the Accuracy-aware Quantization algorithm is less than after the Default Quantization when the model gets fully quantized. + +### Reducing the performance gap of Accuracy-aware Quantization +To improve model performance after Accuracy-aware Quantization, you can try the `"tune_hyperparams"` setting and set it to `True`. It will enable searching for optimal quantization parameters before reverting layers to the "backup" precision. Note, that this can increase the overall quantization time. If you do not achieve the desired accuracy and performance after applying the -`AccuracyAwareQuantization` algorithm or you need an accurate fully-quantized model, -we recommend either using layer-wise hyperparameters tuning with TPE or using -Quantization-Aware training from [the supported frameworks](LowPrecisionOptimizationGuide.md). +Accuracy-aware Quantization algorithm or you need an accurate fully-quantized model, we recommend either using Quantization-Aware Training from [NNCF](@ref docs_nncf_introduction). diff --git a/tools/pot/docs/CLI.md b/tools/pot/docs/CLI.md index f1485bad228..fe9b3ef5c04 100644 --- a/tools/pot/docs/CLI.md +++ b/tools/pot/docs/CLI.md @@ -6,38 +6,29 @@ :maxdepth: 1 :hidden: - Simplified mode - End-to-end CLI example + Simplified Mode + pot_configs_README @endsphinxdirective -POT command-line interface (CLI) is designed to optimize models that are supported by the [Accuracy Checker Tool](@ref omz_tools_accuracy_checker) used for accuracy measurement. -If your model is exactly from the OpenVINO™ [Model Zoo](https://github.com/openvinotoolkit/open_model_zoo) or it is similar to one of -its models then you can employ POT CLI to optimize your model. -In other cases, you should consider using POT [API](@ref pot_compression_api_README). To start with POT CLI please refer to the -following [example](@ref pot_configs_examples_README). +## Introduction -Note: There is also the so-called [**Simplified mode**](@ref pot_docs_simplified_mode) aimed at INT8 quantization if the model is from the Computer Vision domain and has a simple dataset preprocessing, like image resize and crop. In this case, you can also use POT CLI for -optimization. However, the accuracy results are not guaranteed in this case. Moreover, you are also limited in the -optimization methods choice since the accuracy measurement is not available. +POT command-line interface (CLI) is aimed at optimizing models that are similar to the models from OpenVINO™ [Model Zoo](https://github.com/openvinotoolkit/open_model_zoo) or if there is a valid [AccuracyChecker Tool](@ref omz_tools_accuracy_checker) configuration file for the model. Examples of AccuracyChecker configuration files can be found on [GitHub](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/public). Each model folder contains YAML configuration file that can be used with POT as is. + +> **NOTE**: There is also the so-called [Simplified mode](@ref pot_docs_simplified_mode) aimed at optimizatoin of models from the Computer Vision domain and has a simple dataset preprocessing, like image resize and crop. In this case, you can also use POT CLI for optimization. However, the accuracy results are not guaranteed in this case. Moreover, you are also limited in the optimization methods choice since the accuracy measurement is not available. - -## Prerequisites -1. Install POT following the [Installation Guide](@ref pot_InstallationGuide). -2. Convert your model from the framework representation into the OpenVINO™ IR format with the -[Model Optimizer](@ref openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide). -3. Prepare the Accuracy Checker configuration file and make sure that the model can be successfully inferred and achieves -similar accuracy numbers as the reference model from the original framework. -4. Activate the Python environment in the command-line shell where the POT and the Accuracy Checker were installed. - ## Run POT CLI There are two ways how to run POT via command line: -- **Basic usage**. In this case you can run POT with basic setting just specifying all the options via command line: +- **Basic usage for DefaultQuantization**. In this case you can run POT with basic setting just specifying all the options via command line. `-q default` stands for [DefaultQuantization](../openvino/tools/pot/algorithms/quantization/default/README.md) algorithm: ```sh pot -q default -m -w --ac-config ``` +- **Basic usage for AccuracyAwareQauntization**. You can also run [AccuracyAwareQuantization](../openvino/tools/pot/algorithms/quantization/accuracy_aware/README.md) method with basic options. `--max-drop 0.01` option defines maximum accuracy deviation to 1 absolute percent from the original model: + ```sh + pot -q accuracy_aware -m -w --ac-config --max-drop 0.01 + ``` - **Advanced usage**. In this case you should prepare a configuration file for the POT where you can specify advanced options for the optimization methods available. See [POT configuration file description](@ref pot_configs_README) for more details. To launch the command-line tool with the configuration file run: diff --git a/tools/pot/docs/DefaultQuantizationUsage.md b/tools/pot/docs/DefaultQuantizationUsage.md new file mode 100644 index 00000000000..548999fba80 --- /dev/null +++ b/tools/pot/docs/DefaultQuantizationUsage.md @@ -0,0 +1,152 @@ +# Quantizing Model {#pot_default_quantization_usage} + +@sphinxdirective + +.. toctree:: + :maxdepth: 1 + :hidden: + + DefaultQuantization Method + +@endsphinxdirective + +## Introduction +This document describes how to apply model quantization with the Default Quantization method without accuracy control using an unannotated dataset. To use this method, you need to create a Python* script using an API of Post-Training Optimization Tool (POT) and implement data preparation logic and quantization pipeline. In case you are not familiar with Python*, you can try [command-line interface](@ref pot_compression_cli_README) of POT which is designed to quantize models from OpenVINO™ [Model Zoo](https://github.com/openvinotoolkit/open_model_zoo). The figure below shows the common workflow of the quantization script implemented with POT API. + +![](./images/default_quantization_flow.png) + +The script should include three basic steps: +1. Prepare data and dataset interface +2. Select quantization parameters +3. Define and run quantization process + +## Prepare data and dataset interface +In most cases, it is required to implement only `openvino.tools.pot.DataLoader` interface which allows acquiring data from a dataset and applying model-specific pre-processing providing access by index. Any implementation should override the following methods: + +- `__len__()`, returns the size of the dataset +- `__getitem__()`, provides access to the data by index in range of 0 to `len(self)`. It also can encapsulate the logic of model-specific pre-processing. The method should return data in the following format: + - `(data, annotation)` + +where `data` is the input that is passed to the model at inference so that it should be properly preprocessed. `data` can be either `numpy.array` object or dictionary, where the key is the name of the model input and value is `numpy.array` which corresponds to this input. Since `annotation` is not used by the Default Quantization method this object can be `None` in this case. + +You can wrap framework data loading classes by `openvino.tools.pot.DataLoader` interface which is usually straightforward. For example, `torch.utils.data.Dataset` has a similar interface as `openvino.tools.pot.DataLoader` so that its TorchVision implementations can be easily wrapped by POT API. + +> **NOTE**: Model-specific preprocessing, for example, mean/scale normalization can be embedded into the model at the conversion step using Model Optimizer component. This should be considered during the implementation of the DataLoader interface to avoid "double" normalization which can lead to the loss of accuracy after optimization. + +The code example below defines `DataLoader` for three popular use cases: images, text, and audio. + +@sphinxtabset + +@sphinxtab{Images} + +@snippet tools/pot/docs/code/data_loaders.py image_loader + +@endsphinxtab + +@sphinxtab{Text} + +@snippet tools/pot/docs/code/data_loaders.py text_loader + +@endsphinxtab + +@sphinxtab{Audio} + +@snippet tools/pot/docs/code/data_loaders.py audio_loader + +@endsphinxtab + + +@endsphinxtabset + +## Select quantization parameters +Default Quantization algorithm has mandatory and optional parameters which are defined as a dictionary: +```python +{ + "name": "DefaultQuantization", + "params": { + "target_device": "ANY", + "stat_subset_size": 300 + }, +} +``` +- `"target_device"` - currently, only two options are available: `"ANY"` (or `"CPU"`) - to quantize model for CPU, GPU, or VPU, and `"GNA"` - for inference on GNA. +- `"stat_subset_size"` - size of data subset to calculate activations statistics used for quantization. The whole dataset is used if no parameter specified. We recommend using not less than 300 samples. + +Full specification of the Default Quantization method is available in this [document](@ref pot_compression_algorithms_quantization_default_README). + +## Run quantization +POT API provides its own methods to load and save model objects from OpenVINO Intermediate Representation: `load_model` and `save_model`. It also has a concept of `Pipeline` that sequentially applies specified optimization methods to the model. `create_pipeine` method is used to instantiate a `Pipeline` object. +A code example below shows a basic quantization workflow: + +```python +from openvino.tools.pot import IEEngine +from openvino.tools.pot load_model, save_model +from openvino.tools.pot import compress_model_weights +from openvino.tools.pot import create_pipeline + +# Model config specifies the model name and paths to model .xml and .bin file +model_config = +{ + "model_name": "model", + "model": path_to_xml, + "weights": path_to_bin, +} + +# Engine config +engine_config = {"device": "CPU"} + +algorithms = [ + { + "name": "DefaultQuantization", + "params": { + "target_device": "ANY", + "stat_subset_size": 300 + }, + } +] + +# Step 1: Implement and create user's data loader +data_loader = ImageLoader("") + +# Step 2: Load model +model = load_model(model_config=model_config) + +# Step 3: Initialize the engine for metric calculation and statistics collection. +engine = IEEngine(config=engine_config, data_loader=data_loader) + +# Step 4: Create a pipeline of compression algorithms and run it. +pipeline = create_pipeline(algorithms, engine) +compressed_model = pipeline.run(model=model) + +# Step 5 (Optional): Compress model weights to quantized precision +# to reduce the size of the final .bin file. +compress_model_weights(compressed_model) + +# Step 6: Save the compressed model to the desired path. +# Set save_path to the directory where the model should be saved +compressed_model_paths = save_model( + model=compressed_model, + save_path="optimized_model", + model_name="optimized_model", +) +``` + +The output of the script is the quantized model that can be used for inference in the same way as the original full-precision model. + +If accuracy degradation after applying the Default Quantization method is high, it is recommended to try tips from [Quantization Best Practices](@ref pot_docs_BestPractices) document or use [Accuracy-aware Quantization](@ref pot_accuracyaware_usage) method. + +## Quantizing cascaded models +In some cases, when the optimizing model is a cascaded model, i.e. consists of several submodels, for example, MT-CNN, you will need to implement a complex inference pipeline that can properly handle different submodels and data flow between them. POT API provides an `Engine` interface for this purpose which allows customization of the inference logic. However, we suggest inheriting from `IEEngine` helper class that already contains all the logic required to do the inference based on OpenVINO™ Python API. See the following [example](@ref pot_example_face_detection_README). + +## Examples + +* Tutorials: + * [Quantization of Image Classification model](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/301-tensorflow-training-openvino) + * [Quantization of Object Detection model from Model Zoo](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/111-detection-quantization) + * [Quantization of Segmentation model for medical data](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/110-ct-segmentation-quantize) + * [Quantization of BERT for Text Classification](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/105-language-quantize-bert) +* Samples: + * [Quantization of 3D segmentation model](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/3d_segmentation) + * [Quantization of Face Detection model](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/face_detection) + * [Quantizatin of speech model for GNA device](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/speech) + diff --git a/tools/pot/docs/E2eExample.md b/tools/pot/docs/E2eExample.md index 7f7186f2f65..775ee527b4a 100644 --- a/tools/pot/docs/E2eExample.md +++ b/tools/pot/docs/E2eExample.md @@ -1,11 +1,11 @@ -# End-to-end Command-line Interface example {#pot_configs_examples_README} +# End-to-end Command-line Interface Example {#pot_configs_examples_README} This tutorial describes an example of running post-training quantization for **MobileNet v2 model from PyTorch** framework, particularly by the DefaultQuantization algorithm. The example covers the following steps: - Environment setup - Model preparation and converting it to the OpenVINO™ Intermediate Representation (IR) format -- Performance benchmarking of the original full-precision model and the converted one to the IR +- Performance benchmarking of the original full-precision model - Dataset preparation - Accuracy validation of the full-precision model in the IR format - Model quantization by the DefaultQuantization algorithm and accuracy validation of the quantized model @@ -18,31 +18,19 @@ The example has been verified in Ubuntu 18.04 Operating System with Python 3.6 i In case of issues while running the example, refer to [POT Frequently Asked Questions](@ref pot_docs_FrequentlyAskedQuestions) for help. -## Environment Setup - -1. Install OpenVINO™ toolkit and Model Optimizer, Accuracy Checker and Post-training Optimization Tool components following the [Installation Guide](@ref pot_InstallationGuide). -2. Activate the Python* environment and OpenVINO environment as described in the [Installation Guide](@ref pot_InstallationGuide). -3. Create a separate working directory and navigate to it. - -In the instructions below, the Post-Training Optimization Tool directory `` is referred to: - - `/lib/python/site-packages/` in the case of PyPI installation, where `` is a Python* - environment where OpenVINO is installed and `` is a Python* version, e.g. `3.6`. - -`` is the directory where Intel® Distribution of OpenVINO™ toolkit is installed. - ## Model Preparation 1. Navigate to ``. 2. Download the MobileNet v2 PyTorch model using [Model Downloader](@ref omz_tools_downloader) tool from the Open Model Zoo repository: ```sh - python3 ./downloader.py --name mobilenet-v2-pytorch + omz_downloader --name mobilenet-v2-pytorch ``` After that the original full-precision model is located in `/public/mobilenet-v2-pytorch/`. 3. Convert the model to the OpenVINO™ Intermediate Representation (IR) format using [Model Converter](@ref omz_tools_downloader) tool: ```sh - python3 ./converter.py --name mobilenet-v2-pytorch + omz_converter --name mobilenet-v2-pytorch ``` After that the full-precision model in the IR format is located in `/public/mobilenet-v2-pytorch/FP32/`. @@ -50,19 +38,9 @@ For more information about the Model Optimizer, refer to its [documentation](@re ## Performance Benchmarking of Full-Precision Models -1. Check the performance of the original model using [Deep Learning Benchmark](@ref openvino_inference_engine_tools_benchmark_tool_README) tool: +Check the performance of the full-precision model in the IR format using [Deep Learning Benchmark](@ref openvino_inference_engine_tools_benchmark_tool_README) tool: ```sh - python3 ./benchmark_app.py -m /public/mobilenet-v2-pytorch/mobilenet-v2.onnx - ``` - Note that the results might be different dependently on characteristics of your machine. On a machine with Intel® Core™ i9-10920X CPU @ 3.50GHz it is like: - ```sh - Latency: 4.09 ms - Throughput: 1456.84 FPS - ``` - -2. Check the performance of the full-precision model in the IR format using [Deep Learning Benchmark](@ref openvino_inference_engine_tools_benchmark_tool_README) tool: - ```sh - python3 ./benchmark_app.py -m /public/mobilenet-v2-pytorch/FP32/mobilenet-v2-pytorch.xml + benchmark_app -m /public/mobilenet-v2-pytorch/FP32/mobilenet-v2-pytorch.xml ``` Note that the results might be different dependently on characteristics of your machine. On a machine with Intel® Core™ i9-10920X CPU @ 3.50GHz it is like: ```sh @@ -190,7 +168,7 @@ specify the full-precision model in the IR format, `"config": "./mobilenet_v2_py Check the performance of the quantized model using [Deep Learning Benchmark](@ref openvino_inference_engine_tools_benchmark_tool_README) tool: ```sh -python3 ./benchmark_app.py -m +benchmark_app -m ``` where `` is the path to the quantized model. Note that the results might be different dependently on characteristics of your machine. On a machine with Intel® Core™ i9-10920X CPU @ 3.50GHz it is like: diff --git a/tools/pot/docs/Examples.md b/tools/pot/docs/Examples.md new file mode 100644 index 00000000000..6563447772e --- /dev/null +++ b/tools/pot/docs/Examples.md @@ -0,0 +1,17 @@ +# Examples {#pot_examples_description} + +@sphinxdirective + +.. toctree:: + :maxdepth: 1 + :hidden: + + API Examples + Command-line Example + +@endsphinxdirective + +This section provides a set of examples that demonstrate how to apply the post-training optimization methods to optimize various models from different domains. It contains optimization recipes for concrete models, that unnecessarily cover your case, but which should be sufficient to reuse these recipes to optimize custom models: + +- [API Examples](@ref pot_example_README) +- [Commanad-line Example](@ref pot_configs_examples_README) \ No newline at end of file diff --git a/tools/pot/docs/FrequentlyAskedQuestions.md b/tools/pot/docs/FrequentlyAskedQuestions.md index 0dbd9720606..494d196e5bc 100644 --- a/tools/pot/docs/FrequentlyAskedQuestions.md +++ b/tools/pot/docs/FrequentlyAskedQuestions.md @@ -17,6 +17,7 @@ What else can I do? - When I execute POT CLI, I get "File "/workspace/venv/lib/python3.6/site-packages/nevergrad/optimization/base.py", line 35... SyntaxError: invalid syntax". What is wrong? - What does a message "ModuleNotFoundError: No module named 'some\_module\_name'" mean? - Is there a way to collect an intermidiate IR when the AccuracyAware mechanism fails? +- What do the messages "Output name: not found" or "Output node with is not found in graph" mean? ### Is the Post-training Optimization Tool (POT) opensourced? @@ -26,9 +27,7 @@ Yes, POT is developed on GitHub as a part of [https://github.com/openvinotoolkit ### Can I quantize my model without a dataset? In general, you should have a dataset. The dataset should be annotated if you want to validate the accuracy. -If your dataset is not annotated, you can still quantize the model in the Simplified mode but you will not be able to measure the accuracy. -See [Post-Training Optimization Best Practices](BestPractices.md) for more details. -You can also use [POT API](../openvino/tools/pot/api/README.md) to integrate the post-training quantization into the custom inference pipeline. +If your dataset is not annotated, you can use [Default Quantization](@ref pot_default_quantization_usage) to quantize the model or command-line interface with [Simplified mode](@ref pot_docs_simplified_mode). ### Can a model in any framework be quantized by the POT? @@ -37,10 +36,11 @@ The POT accepts models in the OpenVINO™ Intermediate Representation (IR) f ### I'd like to quantize a model and I've converted it to IR but I don't have the Accuracy Checker config. What can I do? -To create the Accuracy Checker configuration file, refer to [Accuracy Checker documentation](@ref omz_tools_accuracy_checker) and -try to find the configuration file for your model among the ones available in the Accuracy Checker examples. An alternative way is to quantize the model -in the Simplified mode but you will not be able to measure the accuracy. See [Post-Training Optimization Best Practices](BestPractices.md) for more details. -Also, you can use [POT API](../openvino/tools/pot/api/README.md) to integrate the post-training quantization into your pipeline without the Accuracy Checker. +1. Try quantization using Python* API of the Post-training Optimization Tool. For more details see [Default Quantization](@ref pot_default_quantization_usage. +2. If you consider command-line usage only refer to [Accuracy Checker documentation](@ref omz_tools_accuracy_checker) to create the Accuracy Checker configuration file, and +try to find the configuration file for your model among the ones available in the Accuracy Checker examples. +3. An alternative way is to quantize the model +in the [Simplified mode](#ref pot_docs_simplified_mode) but you will not be able to measure the accuracy. ### What is a tradeoff when you go to low precision? @@ -53,10 +53,10 @@ The other benefit of having a model in low precision is its smaller size. First of all, you should validate the POT compression pipeline you are running, which can be done with the following steps: 1. Make sure the accuracy of the original uncompressed model has the value you expect. Run your POT pipeline with an empty compression config and evaluate the resulting model metric. Compare this uncompressed model accuracy metric value with your reference. -2. Run your compression pipeline with a single compression algorithm ([DefaultQuantization](../openvino/tools/pot/algorithms/quantization/default/README.md) or [AccuracyAwareQuantization](../openvino/tools/pot/algorithms/quantization/accuracy_aware/README.md)) without any parameter values specified in the config (except for `preset` and `stat_subset_size`). Make sure you get the undesirable accuracy drop/performance gain in this case. +2. Run your compression pipeline with a single compression algorithm ([Default Quantization](@ref pot_default_quantization_usage) or [Accuracy-aware Quantization](@ref pot_accuracyaware_usage)) without any parameter values specified in the config (except for `preset` and `stat_subset_size`). Make sure you get the desirable accuracy drop/performance gain in this case. Finally, if you have done the steps above and the problem persists, you could try to compress your model using the [Neural Network Compression Framework (NNCF)](https://github.com/openvinotoolkit/nncf_pytorch). -Note that NNCF usage requires you to have a PyTorch-based training pipeline of your model in order to perform compression-aware fine-tuning. See [Low Precision Optimization Guide](LowPrecisionOptimizationGuide.md) for more details. +Note that NNCF usage requires you to have a PyTorch or TensorFlow 2 based training pipeline of your model to perform Quantization-aware Training. See [Model Optimization Guide](@ref openvino_docs_model_optimization_guide) for more details. ### I get “RuntimeError: Cannot get memory” and “RuntimeError: Output data was not allocated” when I quantize my model by the POT. @@ -73,12 +73,13 @@ which is usually more accurate and takes more time but requires less memory. See It can happen due to the following reasons: - A wrong or not representative dataset was used during the quantization and accuracy validation. Please make sure that your data and labels are correct and they sufficiently reflect the use case. -- A wrong Accuracy Checker configuration file was used during the quantization. Refer to [Accuracy Checker documentation](@ref omz_tools_accuracy_checker) for more information. +- If the command-line interface was used for quantization, a wrong Accuracy Checker configuration file could lead to this problem. Refer to [Accuracy Checker documentation](@ref omz_tools_accuracy_checker) for more information. +- If [Default Quantization](@ref pot_default_quantization_usage) was used for quantization you can also try [Accuracy-aware Quantization](@ref pot_accuracyaware_usage) method that allows controlling maximum accuracy deviation. ### The quantization process of my model takes a lot of time. Can it be decreased somehow? Quantization time depends on multiple factors such as the size of the model and the dataset. It also depends on the algorithm: -the [DefaultQuantization](../openvino/tools/pot/algorithms/quantization/default/README.md) algorithm takes less time than the [AccuracyAwareQuantization](../openvino/tools/pot/algorithms/quantization/accuracy_aware/README.md) algorithm. +the [Default Quantization](@ref pot_default_quantization_usage) algorithm takes less time than the [ [Accuracy-aware Quantization](@ref pot_accuracyaware_usage) algorithm. The following configuration parameters also impact the quantization time duration (see details in [Post-Training Optimization Best Practices](BestPractices.md)): - `use_fast_bias`: when set to `false`, it increases the quantization time @@ -88,18 +89,9 @@ The following configuration parameters also impact the quantization time duratio - `eval_requests_number`: the lower number, the more time might be required for the quantization Note that higher values of `stat_requests_number` and `eval_requests_number` increase memory consumption by POT. -### I get "Import Error:... No such file or directory". How can I avoid it? - -It happens when some needed library is not available in your environment. To avoid it, execute the following command: -```sh -source /bin/setupvars.sh -``` -where `` is the directory where the OpenVINO™ toolkit is installed. - ### When I execute POT CLI, I get "File "/workspace/venv/lib/python3.6/site-packages/nevergrad/optimization/base.py", line 35... SyntaxError: invalid syntax". What is wrong? -This error is reported when you have an older python version than 3.6 in your environment. Upgrade your python version. Refer to more details about the prerequisites -on the [Post-Training Optimization Tool](../README.md) page. +This error is reported when you have a Python version older than 3.6 in your environment. Upgrade your Python version. ### What does a message "ModuleNotFoundError: No module named 'some\_module\_name'" mean? @@ -108,3 +100,6 @@ It means that some required python module is not installed in your environment. ### Is there a way to collect an intermidiate IR when the AccuracyAware mechanism fails? You can add `"dump_intermediate_model": true` to the POT configuration file and it will drop an intermidiate IR to `accuracy_aware_intermediate` folder. + +### What do the messages "Output name: not found" or "Output node with is not found in graph" mean? +Errors are caused by missing output nodes names in a graph when using the POT tool for model quantization. It might appear for some models only for IRs converted from ONNX models using new frontend (which is the default conversion path starting from 2022.1 release). To avoid such errors, use legacy MO frontend to convert a model to IR by passing the --use_legacy_frontend option. Then, use the produced IR for quantization. diff --git a/tools/pot/docs/InstallationGuide.md b/tools/pot/docs/InstallationGuide.md index 246122520c5..5fb7ab31c09 100644 --- a/tools/pot/docs/InstallationGuide.md +++ b/tools/pot/docs/InstallationGuide.md @@ -1,17 +1,15 @@ -# Post-Training Optimization Tool Installation Guide {#pot_InstallationGuide} - -## Prerequisites - -* Python* 3.6 or higher -* [OpenVINO™](https://docs.openvino.ai/latest/index.html) - -The minimum and the recommended requirements to run the Post-training Optimization Tool (POT) are the same as in [OpenVINO™](https://docs.openvino.ai/latest/index.html). - +# Installation Guide ## Install POT from PyPI -The simplest way to get the Post-training Optimization Tool and OpenVINO™ installed is to use PyPI. Follow the steps below to do that: -1. Create a separate [Python* environment](https://docs.python.org/3/tutorial/venv.html) and activate it -2. To install OpenVINO™, run `pip install openvino`. -3. To install POT and other OpenVINO™ developer tools, run `pip install openvino-dev`. +POT is distributed as a part of OpenVINO™ Development Tools package. For installation instruction, refer to this [document](@ref openvino_docs_install_guides_install_dev_tools). -Now the Post-training Optimization Tool is available in the command line by the `pot` alias. To verify it, run `pot -h`. +## Install POT from GitHub +The latest version of the Post-training Optimization Tool is available on [GitHub](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot) and can be installed from source. As prerequisites, you need to install [OpenVINO™ Runtime](@ref openvino_docs_install_guides_install_runtime) and other dependencies such as [Model Optimizer](@ref openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide) and [Accuracy Checker](@ref omz_tools_accuracy_checker). + +To install POT from source: +- Clone the OpenVINO repository + ```sh + git clone --recusive https://github.com/openvinotoolkit/openvino.git + ``` + +After installation, POT is available as a Python library under `openvino.tools.pot.*` and in the command line by the `pot` alias. To verify it, run `pot -h`. diff --git a/tools/pot/docs/Introduction.md b/tools/pot/docs/Introduction.md new file mode 100644 index 00000000000..d1cb3e61a8e --- /dev/null +++ b/tools/pot/docs/Introduction.md @@ -0,0 +1,51 @@ +# Optimizing models post-training {#pot_introduction} + +@sphinxdirective + +.. toctree:: + :maxdepth: 1 + :hidden: + + Quantizing Model + Quantizing Model with Accuracy Control + Quantization Best Practices + API Reference + Command-line Interface + Examples + pot_docs_FrequentlyAskedQuestions + +@endsphinxdirective + +## Introduction + +Post-training model optimization is the process of applying special methods without model retraining or fine-tuning, for example, post-training 8-bit quantization. Therefore, this process does not require a training dataset or a training pipeline in the source DL framework. To apply post-training methods in OpenVINO™, you need: +* A floating-point precision model, FP32 or FP16, converted into the OpenVINO™ Intermediate Representation (IR) format +and run on CPU with the OpenVINO™. +* A representative calibration dataset representing a use case scenario, for example, 300 samples. +* In case of accuracy constraints, a validation dataset and accuracy metrics should be available. + +For the needs of post-training optimization, OpenVINO™ provides a Post-training Optimization Tool (POT) which supports the uniform integer quantization method. This method allows substantially increasing inference performance and reducing the model size. + +Figure below shows the optimization workflow with POT: +![](./images/workflow_simple.png) + + +## Quantizing models with POT + +POT provides two main quantization methods that can be used depending on the user's needs and requirements: + +* [Default Quantization](@ref pot_default_quantization_usage) is a recommended method that provides fast and accurate results in most cases. It requires only a unannotated dataset for quantization. For details, see the [Default Quantization algorithm](@ref pot_compression_algorithms_quantization_default_README) documentation. + +* [Accuracy-aware Quantization](@ref pot_accuracyaware_usage) is an advanced method that allows keeping accuracy at a predefined range at the cost of performance improvement in case when `Default Quantization` cannot guarantee it. The method requires annotated representative dataset and may require more time for quantization. For details, see the +[Accuracy-aware Quantization algorithm](@ref accuracy_aware_README) documentation. + +HW platforms support different integer precisions and quantization parameters, for example 8-bit in CPU, GPU, VPU, 16-bit for GNA. POT abstracts this complexity by introducing a concept of "target device" that is used to set quantization settings specific to the device. The `target_device` parameter is used for this purpose. + +> **NOTE**: There is a special `target_device: "ANY"` which leads to portable quantized models compatible with CPU, GPU, and VPU devices. GNA-quantized models are compatible only with CPU. + +For benchmarking results collected for the models optimized with the POT tool, refer to [INT8 vs FP32 Comparison on Select Networks and Platforms](@ref openvino_docs_performance_int8_vs_fp32). + +## See Also + +* [Performance Benchmarks](https://docs.openvino.ai/latest/openvino_docs_performance_benchmarks_openvino.html) +* [INT8 Quantization by Using Web-Based Interface of the DL Workbench](https://docs.openvino.ai/latest/workbench_docs_Workbench_DG_Int_8_Quantization.html) diff --git a/tools/pot/docs/LowPrecisionOptimizationGuide.md b/tools/pot/docs/LowPrecisionOptimizationGuide.md index 493ae6bbe3f..82f200fbb95 100644 --- a/tools/pot/docs/LowPrecisionOptimizationGuide.md +++ b/tools/pot/docs/LowPrecisionOptimizationGuide.md @@ -1,4 +1,4 @@ -# Low Precision Optimization Guide {#pot_docs_LowPrecisionOptimizationGuide} +# Low Precision Optimization Guide ## Introduction This document provides the best-known methods on how to use low-precision capabilities of the OpenVINO™ toolkit to transform models diff --git a/tools/pot/docs/ModelRepresentation.md b/tools/pot/docs/ModelRepresentation.md index c5c78367046..052dbfbeffd 100644 --- a/tools/pot/docs/ModelRepresentation.md +++ b/tools/pot/docs/ModelRepresentation.md @@ -1,4 +1,4 @@ -# Low-precision model representation {#pot_docs_model_representation} +# Low-precision model representation ## Introduction The goal of this document is to describe how optimized models are represented in OpenVINO Intermediate Representation (IR) and provide guidance on interpretation rules for such models at runtime. diff --git a/tools/pot/docs/SaturationIssue.md b/tools/pot/docs/SaturationIssue.md index 61e5e1a117a..c74cf993a3a 100644 --- a/tools/pot/docs/SaturationIssue.md +++ b/tools/pot/docs/SaturationIssue.md @@ -1,4 +1,4 @@ -# Saturation (overflow) issue workaround {#pot_saturation_issue} +# Saturation (overflow) Issue Workaround {#pot_saturation_issue} ## Introduction 8-bit instructions of previous generations of Intel® CPUs, namely those based on SSE, AVX-2, AVX-512 instruction sets, admit so-called saturation (overflow) of the intermediate buffer when calculating the dot product which is an essential part of Convolutional or MatMul operations. This saturation can lead to an accuracy drop on the mentioned architectures during the inference of 8-bit quantized models. However, it is not possible to predict such degradation since most of the computations are executed in parallel during DL model inference which makes this process non-deterministic. This problem is typical for models with non-ReLU activation functions and low level of redundancy, for example, optimized or efficient models. It can prevent deploying the model on legacy hardware or creating cross-platform applications. The problem does not occur on the CPUs with Intel Deep Learning Boost (VNNI) technology and further generations, as well as on GPUs. diff --git a/tools/pot/docs/SimplifiedMode.md b/tools/pot/docs/SimplifiedMode.md index bd574e4ca86..88c2cafbc13 100644 --- a/tools/pot/docs/SimplifiedMode.md +++ b/tools/pot/docs/SimplifiedMode.md @@ -1,4 +1,4 @@ -# Optimization with Simplified mode {#pot_docs_simplified_mode} +# Optimization with Simplified Mode {#pot_docs_simplified_mode} ## Introduction diff --git a/tools/pot/docs/code/data_loaders.py b/tools/pot/docs/code/data_loaders.py new file mode 100644 index 00000000000..9cb358657a1 --- /dev/null +++ b/tools/pot/docs/code/data_loaders.py @@ -0,0 +1,122 @@ +# Copyright (C) 2018-2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +#! [image_loader] +import os + +import numpy as np +import cv2 as cv + +from openvino.tools.pot import DataLoader + +class ImageLoader(DataLoader): + """ Loads images from a folder """ + def __init__(self, dataset_path): + # Use OpenCV to gather image files + # Collect names of image files + self._files = [] + all_files_in_dir = os.listdir(dataset_path) + for name in all_files_in_dir: + file = os.path.join(dataset_path, name) + if cv.haveImageReader(file): + self._files.append(file) + + # Define shape of the model + self._shape = (224,224) + + def __len__(self): + """ Returns the length of the dataset """ + return len(self._files) + + def __getitem__(self, index): + """ Returns image data by index in the NCHW layout + Note: model-specific preprocessing is omitted, consider adding it here + """ + if index >= len(self): + raise IndexError("Index out of dataset size") + + image = cv.imread(self._files[index]) # read image with OpenCV + image = cv.resize(image, self._shape) # resize to a target input size + image = np.expand_dims(image, 0) # add batch dimension + image = image.transpose(0, 3, 1, 2) # convert to NCHW layout + return image, None # annotation is set to None +#! [image_loader] + +#! [text_loader] +import os +from pathlib import Path + +from datasets import load_dataset #pip install datasets +from transformers import AutoTokenizer #pip install transformers + +from openvino.tools.pot import DataLoader + +class TextLoader(DataLoader): + """ Loads content of .txt files from a folder """ + def __init__(self, dataset_path): + # HuggingFace dataset API is used to process text files + # Collect names of text files + extension = ".txt" + files = sorted(str(p.stem) for p in + Path(dataset_path).glob("*" + extension)) + files = [os.path.join(dataset_path, file + extension) for file in files] + self._dataset = load_dataset('text', data_files=files) + # replace with your tokenizer + self._tokenizer = AutoTokenizer.from_pretrained('bert-base-cased') + self._dataset = self._dataset.map(self._encode, batched=False) + # replace with names of model inputs + self._dataset.set_format(type='numpy', + columns=['input_ids', 'token_type_ids', 'attention_mask']) + + def _encode(self, examples): + """ Tokenization of the input text """ + return self._tokenizer(examples['text'], truncation=True, padding='max_length') + + def __len__(self): + """ Returns the length of the dataset """ + return len(self._dataset['train']) + + def __getitem__(self, index): + """ Returns data by index as a (dict[str, np.array], None) """ + if index >= len(self): + raise IndexError("Index out of dataset size") + + data = self._dataset['train'][index] + return {'input_ids': data['input_ids'], + 'token_type_ids': data['token_type_ids'], + 'attention_mask': data['attention_mask']}, None # annotation is set to None +#! [text_loader] + +#! [audio_loader] +import os +from pathlib import Path + +import torchaudio # pip install torch torchaudio + +from openvino.tools.pot import DataLoader + +class AudioLoader(DataLoader): + """ Loads content of .wav files from a folder """ + def __init__(self, dataset_path): + # Collect names of wav files + self._extension = ".wav" + self._dataset_path = dataset_path + self._files = sorted(str(p.stem) for p in + Path(self._dataset_path).glob("*" + self._extension)) + + def __len__(self): + """ Returns the length of the dataset """ + return len(self._files) + + def __getitem__(self, index): + """ Returns wav data by index + Note: model-specific preprocessing is omitted, consider adding it here + """ + if index >= len(self): + raise IndexError("Index out of dataset size") + + file_name = self._files[index] + self._extension + file_path = os.path.join(self._dataset_path, file_name) + waveform, _ = torchaudio.load(file_path) # use a helper from torchaudio to load data + return waveform.numpy(), None # annotation is set to None +#! [audio_loader] \ No newline at end of file diff --git a/tools/pot/docs/images/default_quantization_flow.png b/tools/pot/docs/images/default_quantization_flow.png new file mode 100644 index 00000000000..b69ee969d8d --- /dev/null +++ b/tools/pot/docs/images/default_quantization_flow.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c9a0f1cccf731e0740cb91a3a48ff73b9f54cee0a39a03d421316a5599e7955 +size 37302 diff --git a/tools/pot/docs/pot_docs.xml b/tools/pot/docs/pot_docs.xml index 54c2684b051..efc7c39fa42 100644 --- a/tools/pot/docs/pot_docs.xml +++ b/tools/pot/docs/pot_docs.xml @@ -6,7 +6,7 @@ - + diff --git a/tools/pot/openvino/tools/pot/algorithms/quantization/README.md b/tools/pot/openvino/tools/pot/algorithms/quantization/README.md index 267c85ad915..0444889bf44 100644 --- a/tools/pot/openvino/tools/pot/algorithms/quantization/README.md +++ b/tools/pot/openvino/tools/pot/algorithms/quantization/README.md @@ -1,23 +1,10 @@ -# Quantization {#pot_compression_algorithms_quantization_README} - -@sphinxdirective - -.. toctree:: - :maxdepth: 1 - :hidden: - - DefaultQuantization Algorithm - AccuracyAwareQuantization Algorithm - TunableQuantization Algorithm - Saturation Issue Workaround - Low-precision Model Representation - - -@endsphinxdirective +# Quantization ## Introduction -The primary optimization feature of the Post-training Optimization Tool (POT) is 8-bit uniform quantization which allows substantially increasing inference performance on all the platforms that have 8-bit instructions, for example, modern generations of CPU and GPU. Another benefit of quantization is a significant reduction of model footprint which in most cases achieves 4x. +The primary optimization feature of the Post-training Optimization Tool (POT) is the uniform integer quantization which allows substantially increasing inference performance and reducing the model size. Different HW platforms can support different integer precisions and POT is designed to support all of them, for example, 8-bit for CPU, GPU, VPU, 16-bit for GNA. Moreover, POT makes the specification of HW settings transparent for the user by introducing a concept of the `target_device` parameter. + +> **NOTE**: There is a special `target_device: "ANY"` which leads to portable quantized models compatible with CPU, GPU, and VPU devices. GNA-quantized models are compatible only with CPU. During the quantization process, the POT tool runs inference of the optimizing model to estimate quantization parameters for input activations of the quantizable operation. It means that a calibration dataset is required to perform quantization. This dataset may have or not have annotation depending on the quantization algorithm that is used. @@ -25,12 +12,11 @@ During the quantization process, the POT tool runs inference of the optimizing m Currently, the POT provides two algorithms for 8-bit quantization, which are verified and guarantee stable results on a wide range of DNN models: -* [**DefaultQuantization**](@ref pot_compression_algorithms_quantization_default_README) is a default method that provides fast and in most cases accurate results for 8-bit - quantization. It requires only a unannotated dataset for quantization. For details, see the [DefaultQuantization Algorithm](@ref pot_compression_algorithms_quantization_default_README) documentation. +* [**DefaultQuantization**](@ref pot_compression_algorithms_quantization_default_README) is a default method that provides fast and in most cases accurate results. It requires only an unannotated dataset for quantization. For details, see the [DefaultQuantization Algorithm](@ref pot_compression_algorithms_quantization_default_README) documentation. -* [**AccuracyAwareQuantization**](@ref pot_compression_algorithms_quantization_accuracy_aware_README) enables remaining at a predefined range of accuracy drop after quantization at the cost +* [**AccuracyAwareQuantization**](@ref accuracy_aware_README) enables remaining at a predefined range of accuracy drop after quantization at the cost of performance improvement. The method requires annotated representative dataset and may require more time for quantization. For details, see the - [AccuracyAwareQuantization Algorithm](@ref pot_compression_algorithms_quantization_accuracy_aware_README) documentation. + [AccuracyAwareQuantization Algorithm](@ref accuracy_aware_README) documentation. For more details about the representation of the low-precision model please refer to this [document](@ref pot_docs_model_representation). diff --git a/tools/pot/openvino/tools/pot/algorithms/quantization/accuracy_aware/README.md b/tools/pot/openvino/tools/pot/algorithms/quantization/accuracy_aware/README.md index 2ab67df921d..95ce7f289a5 100644 --- a/tools/pot/openvino/tools/pot/algorithms/quantization/accuracy_aware/README.md +++ b/tools/pot/openvino/tools/pot/algorithms/quantization/accuracy_aware/README.md @@ -1,33 +1,22 @@ -# AccuracyAwareQuantization Algorithm {#pot_compression_algorithms_quantization_accuracy_aware_README} +# AccuracyAwareQuantization Algorithm {#accuracy_aware_README} -## Overview -AccuracyAware algorithm is designed to perform accurate quantization and allows the model to stay in the -pre-defined range of accuracy drop, for example 1%, defined by the user in the configuration file. This may cause a -degradation in performance in comparison to [DefaultQuantization](../default/README.md) algorithm because some layers can be reverted back to the original precision. The algorithm requires annotated dataset and cannot be used with the [Simplified mode](@ref pot_docs_simplified_mode). - -> **NOTE**: In case of GNA `target_device`, POT moves INT8 weights to INT16 to stay in the pre-defined range of the accuracy drop. Thus, the algorithm works for the `performance` (INT8) preset only. For the `accuracy` preset, this algorithm is not performed, but the parameters tuning is available (if `tune_hyperparams` option is enabled). - -Generally, the algorithm consists of the following steps: -1. The model gets fully quantized using the DefaultQuantization algorithm. -2. The quantized and full-precision models are compared on a subset of the validation set in order to find mismatches in the target accuracy metric. A ranking subset is extracted based on the mismatches. -3. Optionally, if the accuracy criteria cannot be satisfied with fully symmetric quantization, the quantized model gets converted to mixed mode, and step 2 is repeated. -4. A layer-wise ranking is performed in order to get a contribution of each quantized layer into the accuracy drop. To -get this ranking we revert every layer (one-by-one) back to floating-point precision and measure how it affects accuracy. -5. Based on the ranking, the most "problematic" layer is reverted back to the original precision. This change is followed by the evaluation of the obtained model on the full validation set in order to get a new accuracy drop. -6. If the accuracy criteria are satisfied for all pre-defined accuracy metrics defined in the configuration file, - the algorithm finishes. Otherwise, it continues reverting the next "problematic" layer. -7. It may happen that regular reverting does not get any accuracy improvement or even worsen the accuracy. Then the -re-ranking is triggered as it is described in step 4. However, it is possible to specify the maximum number of reverting -layers using a special parameter. Moreover, the algorithm saves intermediate results (models) that can be used at any time -without a need to wait until it finishes. - -The figure below shows the diagram of the algorithm. - -![](../../../../../../docs/images/aa_quantization_pipeline.png) +## Introduction +AccuracyAwareQuantization algorithm is aimed at accurate quantization and allows the model's accuracy to stay within the +pre-defined range. This may cause a +degradation in performance in comparison to [DefaultQuantization](../default/README.md) algorithm because some layers can be reverted back to the original precision. ## Parameters -Since the [DefaultQuantization](../default/README.md) algorithm is used as an initialization, all its parameters are also valid and can be specified. Here we -describe only AccuracyAware specific parameters: +Since the [DefaultQuantization](../default/README.md) algorithm is used as an initialization, all its parameters are also valid and can be specified. Here is an example of the definition of the `AccuracyAwareQuantization` method and its parameters: +```json +{ + "name": "AccuracyAwareQuantization", // the name of optimization algorithm + "params": { + ... + } +} +``` + +Below is the description of AccuracyAwareQuantization-specific parameters: - `"ranking_subset_size"` - size of a subset that is used to rank layers by their contribution to the accuracy drop. Default value is `300`. The more samples it has the better ranking you have, potentially. - `"max_iter_num"` - maximum number of iterations of the algorithm, in other words maximum number of layers that may @@ -57,16 +46,10 @@ quantization time. Default value is `False`. ## Examples -A template and full specification for AccuracyAwareQuantization algorithm: - * [Template](https://github.com/openvinotoolkit/openvino/blob/master/tools/pot/openvino/tools/pot/configs/templates/accuracy_aware_quantization_template.json) - * [Full specification](https://github.com/openvinotoolkit/openvino/blob/master/tools/pot/configs/accuracy_aware_quantization_spec.json) - -Example of using POT API with Accuracy-aware algorithm: +Example: * [Quantization of Object Detection model with control of accuracy](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/object_detection) - ## See also -* [Optimization with Simplified mode](@ref pot_docs_simplified_mode) -* [Use POT Command-line for Model Zoo models](@ref pot_compression_cli_README) -* [POT API](@ref pot_compression_api_README) -* [Post-Training Optimization Best Practices](@ref pot_docs_BestPractices) + A template and full specification for AccuracyAwareQuantization algorithm for POT command-line interface: + * [Template](https://github.com/openvinotoolkit/openvino/blob/master/tools/pot/configs/accuracy_aware_quantization_template.json) + * [Full specification](https://github.com/openvinotoolkit/openvino/blob/master/tools/pot/configs/accuracy_aware_quantization_spec.json) diff --git a/tools/pot/openvino/tools/pot/algorithms/quantization/default/README.md b/tools/pot/openvino/tools/pot/algorithms/quantization/default/README.md index 3a8849f38d0..c374633d7cd 100644 --- a/tools/pot/openvino/tools/pot/algorithms/quantization/default/README.md +++ b/tools/pot/openvino/tools/pot/algorithms/quantization/default/README.md @@ -1,35 +1,19 @@ # DefaultQuantization Algorithm {#pot_compression_algorithms_quantization_default_README} -## Overview -DefaultQuantization algorithm is designed to perform a fast and in many cases accurate 8-bits quantization of NNs. -![](../../../../../../docs/images/default_quantization_pipeline.png) -The algorithm consists of three methods that are sequentially applied to a model: -* ActivationChannelAlignment - Used as a preliminary step before quantization and allows you to align ranges of output activations of Convolutional layers in order to reduce the quantization error. -* MinMaxQuantization - This is a vanilla quantization method that automatically inserts [FakeQuantize](@ref openvino_docs_ops_quantization_FakeQuantize_1) operations into the model graph based on the specified target hardware and initializes them -using statistics collected on the calibration dataset. -* FastBiasCorrection - Adjusts biases of Convolutional and Fully-Connected layers based on the quantization error of the layer in order to make the overall error unbiased. - -This algorithm uses a two-stage statistic collection procedure, where the model is being inferred over the calibration -subset, so the wall-time of quantization basically depends on the size of the subset. +## Introduction +DefaultQuantization algorithm is designed to do a fast and, in many cases, accurate quantization. It does not have any control of accuracy metric but provides a lot of knobs that can be used to improve it. ## Parameters -The algorithm accepts all the parameters introduced by three algorithms that it relies on. These parameters should be -described in the corresponding section in the configuration file (see example below): -```json -"compression": { - "algorithms": [ - { - "name": "DefaultQuantization", // the name of optimization algorithm - "params": { - ... - } - } - ] +DefaultQuantization algorithm has mandatory and optional parameters. For more details on how to use these parameters please refer to [Best Practices](@ref pot_docs_BestPractices) document. Here is an example of the definition of DefualtQuantization method and its parameters: +```python +{ + "name": "DefaultQuantization", # the name of optimization algorithm + "params": { + ... + } } ``` -DefaultQuantization algorithm' parameters can be roughly divided into two groups: mandatory and optional. - ### Mandatory parameters - `"preset"` - preset which controls the quantization mode (symmetric and asymmetric). It can take two values: - `"performance"` (default) - stands for symmetric quantization of weights and activations. This is the most @@ -114,28 +98,22 @@ mode on the existing HW. Enabling this option may increase compressed model accuracy, but will result in increased execution time and memory consumption. ## Examples - - A template and full specification for DefaultQuantization algorithm: - * [Template](https://github.com/openvinotoolkit/openvino/blob/master/tools/pot/openvino/tools/pot/configs/templates/default_quantization_template.json) - * [Full specification](https://github.com/openvinotoolkit/openvino/blob/master/tools/pot/configs/default_quantization_spec.json) - -Command-line example: -* [Quantization of Image Classification model](https://docs.openvino.ai/latest/pot_configs_examples_README.html) - -API tutorials: +Tutorials: * [Quantization of Image Classification model](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/301-tensorflow-training-openvino) * [Quantization of Object Detection model from Model Zoo](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/111-detection-quantization) * [Quantization of Segmentation model for mediacal data](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/110-ct-segmentation-quantize) * [Quantization of BERT for Text Classification](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/105-language-quantize-bert) -API examples: +Examples: * [Quantization of 3D segmentation model](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/3d_segmentation) * [Quantization of Face Detection model](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/face_detection) -* [Speech example for GNA device](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/speech) +* [Quantizatin of speech model for GNA device](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/speech) + +Command-line example: +* [Quantization of Image Classification model](https://docs.openvino.ai/latest/pot_configs_examples_README.html) + +A template and full specification for DefaultQuantization algorithm for POT command-line inferface: +* [Template](https://github.com/openvinotoolkit/openvino/blob/master/tools/pot/configs/default_quantization_template.json) +* [Full specification](https://github.com/openvinotoolkit/openvino/blob/master/tools/pot/configs/default_quantization_spec.json) -## See also -* [Optimization with Simplified mode](@ref pot_docs_simplified_mode) -* [Use POT Command-line for Model Zoo models](@ref pot_compression_cli_README) -* [POT API](@ref pot_compression_api_README) -* [Post-Training Optimization Best Practices](@ref pot_docs_BestPractices) diff --git a/tools/pot/openvino/tools/pot/algorithms/quantization/ranger/README.md b/tools/pot/openvino/tools/pot/algorithms/quantization/ranger/README.md index 80fe9ff0c44..aade9beba15 100644 --- a/tools/pot/openvino/tools/pot/algorithms/quantization/ranger/README.md +++ b/tools/pot/openvino/tools/pot/algorithms/quantization/ranger/README.md @@ -1,8 +1,7 @@ -# Experimental: Deep neural network protection through range supervision ("Ranger") {#pot_ranger_README} +# Experimental: Protecting Deep Learning Model through Range Supervision ("Ranger") {#pot_ranger_README} -# Overview +## Introdution -## Method and workflow Deep neural network find applications in many scenarios where the prediction is a critical component for safety-relevant decisions. Such workloads can benefit from additional protection against underlying errors. For example, memory bit flips (**"soft errors"** originating, e.g., from external radiation or internal electrical disturbances within the circuitry) in der platform hosting the network inference can corrupt the learned network parameters and lead to incorrect predictions. Typically, errors resulting in very large parameter values have a more drastic impact on the network behavior. **The range supervision algorithm ("Ranger") described here establishes and inserts additional protection layers after already present activation layers**. Those layers truncate values that are found to be out of an expected activation range in order to mitigate the traces of potential platform errors. They do so during inference by applying a *clamp* operation to any activation *x* in the input to the Ranger layer, ```math @@ -18,18 +17,8 @@ The process flow follows the diagram [Fig 1](#Schematic). Starting from the inte *Fig 1: Schematic of Ranger process flow.* -## Example -The following example shows a traffic camera image and predicted objects using a Yolov3 pretrained on the Coco dataset. A single weight fault was injected in a randomly chosen convolution layer of Yolo, flipping the most significant bit of the selected network parameter. If range supervision is applied, the original network performance is recovered despite the presence of the fault. - -![](../../../../../../docs/ranger/images/img_combined_2.png) - -*Fig 2: Example of fault mitigation via range supervision.* - - -# Usage - -## Supported activation layers +### Supported activation layers The following activation layers are currently supported for range supervision: @@ -43,42 +32,36 @@ The following activation layers are currently supported for range supervision: This means that any activation layer of one of the above types, that the model under consideration contains, will be protected with an appropriate subsequent Ranger layer. -## Algorithm configuration -The range supervision algorithm is part of the post-training-optimizer tool and can be called by +## Usage +Ranger protection can be used the same way as [DefaultQuantization](@ref pot_default_quantization_usage) method. - pot -c config_file - -Here, config_file is a separate file that has to be created to specify all details of the process. The minimal structure is (here in json format): +### Algorithm configuration +Algorithm has a minimal configuration. Below is an example of such configuration: ```json { - "compression": { - "algorithms": [ - { - "name": "Ranger", - "params": { - "stat_subset_size": 300 - } - - } - ] - ... - } - ... + "name": "Ranger", + "params": { + "stat_subset_size": 300 + } + } ``` The protected model will be saved in IR format in a new folder *./results/\_Ranger/...* . Mandatory parameters: +- `"stat_subset_size"`: This parameter defines *how many images* of the specified dataset in "engine: config" are used to extract the bounds (images are randomly chosen if a subset is chosen). This value is set to **300** by default. The more images are selected for the bound generation, the more accurate the estimation of an out-of-bound event will be, at the cost of increasing extraction time. -- `"compression"` : - - `"algorithms"`: - - `"name"`: Algorithm name, here **"Ranger"**. - - `"params"`: - - `"stat_subset_size"`: This parameter defines *how many images* of the specified dataset in "engine: config" are used to extract the bounds (images are randomly chosen if a subset is chosen). This value is set to **300** by default. The more images are selected for the bound generation, the more accurate the estimation of an out-of-bound event will be, at the cost of increasing extraction time. +## Example of Ranger results +The following example shows a traffic camera image and predicted objects using a Yolov3 pretrained on the Coco dataset. A single weight fault was injected in a randomly chosen convolution layer of Yolo, flipping the most significant bit of the selected network parameter. If range supervision is applied, the original network performance is recovered despite the presence of the fault. -# Resources: + +![](../../../../../../docs/ranger/images/img_combined_2.png) + +*Fig 2: Example of fault mitigation via range supervision.* + +## Resources: - Z. Chen, G. Li, and K. Pittabiraman, "A Low-cost Fault Corrector for Deep Neural Networks through Range Restriction", 2020. https://arxiv.org/abs/2003.13874 - F. Geissler, Q. Syed, S. Roychowdhury, A. Asgari, Y. Peng, A. Dhamasia, R. Graefe, K. Pattabiraman, and M. Paulitsch, "Towards a Safety Case for Hardware Fault Tolerance in Convolutional Neural Networks Using Activation Range Supervision", 2021. https://arxiv.org/abs/2108.07019 diff --git a/tools/pot/openvino/tools/pot/algorithms/quantization/tunable_quantization/README.md b/tools/pot/openvino/tools/pot/algorithms/quantization/tunable_quantization/README.md index 38033ce5fd4..1197448c466 100644 --- a/tools/pot/openvino/tools/pot/algorithms/quantization/tunable_quantization/README.md +++ b/tools/pot/openvino/tools/pot/algorithms/quantization/tunable_quantization/README.md @@ -1,4 +1,4 @@ -# TunableQuantization Algorithm {#pot_compression_algorithms_quantization_tunable_quantization_README} +# TunableQuantization Algorithm ## Overview diff --git a/tools/pot/openvino/tools/pot/api/README.md b/tools/pot/openvino/tools/pot/api/README.md index 06b9af6a165..8c0399ed7cb 100644 --- a/tools/pot/openvino/tools/pot/api/README.md +++ b/tools/pot/openvino/tools/pot/api/README.md @@ -1,74 +1,6 @@ -# Post-Training Optimization Tool API {#pot_compression_api_README} +# API Reference {#pot_compression_api_README} -@sphinxdirective - -.. toctree:: - :maxdepth: 1 - :hidden: - - API Samples - -@endsphinxdirective - -## Overview -The Post-Training Optimization Tool (POT) Python* API allows injecting optimization methods supported by POT into a -model inference script written with OpenVINO™ [Python* API](ie_python_api/api.html). -Thus, POT API helps to implement a custom -optimization pipeline for a single or cascaded/composite DL model (set of joint models). By the optimization pipeline, -we mean the consecutive application of optimization algorithms to the model. The input for the optimization pipeline is -a full-precision model, and the result is an optimized model. The optimization pipeline is configured to sequentially -apply optimization algorithms in the order they are specified. The key requirement for applying the optimization -algorithm is the availability of the calibration dataset for statistics collection and validation dataset for accuracy -validation which in practice can be the same. The Python* POT API provides simple interfaces for implementing: -- custom model inference pipeline with OpenVINO Inference Engine, -- data loading and pre-processing on an arbitrary dataset, -- custom accuracy metrics, - -to make it possible to use optimization algorithms from the POT. - -The Python* POT API provides `Pipeline` class for creating and configuring the optimization pipeline and applying it to -the model. The `Pipeline` class depends on the implementation of the following model specific interfaces which -should be implemented according to the custom DL model: -- `Engine` is responsible for model inference and provides statistical data and accuracy metrics for the model. - > **NOTE**: The POT has the implementation of the Engine class with the class name IEEngine located in - > `/engines/ie_engine.py`, where `` is a directory where the Post-Training Optimization Tool is installed. - > It is based on the [OpenVINO™ Inference Engine Python* API](ie_python_api/api.html) - > and can be used as a baseline engine in the customer pipeline instead of the abstract Engine class. -- `DataLoader` is responsible for the dataset loading, including the data pre-processing. -- `Metric` is responsible for calculating the accuracy metric for the model. - > **NOTE**: Metric is required if you want to use accuracy-aware optimization algorithms, such as `AccuracyAwareQuantization` - > algorithm. - -The pipeline with implemented model specific interfaces such as `Engine`, `DataLoader` and `Metric` we will call the custom -optimization pipeline (see the picture below that shows relationships between classes). - -![](../../../../docs/images/api.png) - -## Use Cases -Before diving into the Python* POT API, it is highly recommended to read [Best Practices](@ref pot_docs_BestPractices) document where various -scenarios of using the Post-Training Optimization Tool are described. - -The POT Python* API for model optimization can be used in the following cases: -- [Accuracy Checker](@ref omz_tools_accuracy_checker) tool does not support the model or dataset. -- POT does not support the model in the [Simplified Mode](@ref pot_docs_BestPractices) or produces the optimized model with low -accuracy in this mode. -- You already have the Python* script to validate the accuracy of the model using the [OpenVINO™ Runtime](@ref openvino_docs_OV_Runtime_User_Guide). - -## Examples - -* API tutorials: - * [Quantization of Image Classification model](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/301-tensorflow-training-openvino) - * [Quantization of Object Detection model from Model Zoo](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/111-detection-quantization) - * [Quantization of BERT for Text Classification](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/105-language-quantize-bert) -* API examples: - * [Quantization of 3D segmentation model](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/3d_segmentation) - * [Quantization of Face Detection model](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/face_detection) - * [Speech example for GNA device](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/speech) - -## API Description - -Below is a detailed explanation of POT Python* APIs which should be implemented in order to create a custom optimization -pipeline. +Post-training Optimization Tool API provides a full set of interfaces and helpers that allow users to implement a custom optimization pipeline for various types of DL models including cascaded or compound models. Below is a full specification of this API: ### DataLoader @@ -81,7 +13,15 @@ The base class for all DataLoaders. by index. All subclasses should override `__len__()` function, which should return the size of the dataset, and `__getitem__()`, -which supports integer indexing in range of 0 to `len(self)` +which supports integer indexing in the range of 0 to `len(self)`. `__getitem__()` method can return data in one of the possible formats: +``` +(data, annotation) +``` +or +``` +(data, annotation, metadata) +``` +`data` is the input that is passed to the model at inference so that it should be properly preprocessed. `data` can be either `numpy.array` object or dictionary where the key is the name of the model input and value is `numpy.array` which corresponds to this input. The format of `annotation` should correspond to the expectations of the `Metric` class. `metadata` is an optional field that can be used to store additional information required for post-processing. ### Metric @@ -90,10 +30,15 @@ class openvino.tools.pot.Metric() ``` An abstract class representing an accuracy metric. -All subclasses should override the following properties: -- `value` - returns the accuracy metric value for the last model output. -- `avg_value` - returns the average accuracy metric value for all model outputs. -- `attributes` - returns a dictionary of metric attributes: +All instances should override the following properties: +- `value` - returns the accuracy metric value for the last model output in a format of `Dict[str, numpy.array]`. +- `avg_value` - returns the average accuracy metric over collected model results in a format of `Dict[str, numpy.array]`. +- `higher_better` should return `True` if a higher value of the metric corresponds to better performance, otherwise, returns `False`. Default implementation returns `True`. + +and methods: +- `update(output, annotation)` - calculates and updates the accuracy metric value using the last model output and annotation. The model output and annotation should be passed in this method. It should also contain the model-specific post-processing in case the model returns the raw output. +- `reset()` - resets collected accuracy metric. +- `get_attributes()` - returns a dictionary of metric attributes: ``` {metric_name: {attribute_name: value}} ``` @@ -102,10 +47,6 @@ All subclasses should override the following properties: should be increased in accuracy-aware algorithms. - `type` - a string representation of metric type. For example, 'accuracy' or 'mean_iou'. -All subclasses should override the following methods: -- `update(output, annotation)` - calculates and updates the accuracy metric value using last model output and annotation. -- `reset()` - resets collected accuracy metric. - ### Engine ``` @@ -232,11 +173,11 @@ The following methods can be overridden in subclasses: `IEEngine` supports data returned by `DataLoader` in the format: ``` -(img_id, img_annotation), image) +(data, annotation) ``` or ``` -((img_id, img_annotation), image, image_metadata) +(data, annotation, metadata) ``` Metric values returned by a `Metric` instance are expected to be in the format: diff --git a/tools/pot/openvino/tools/pot/api/samples/3d_segmentation/README.md b/tools/pot/openvino/tools/pot/api/samples/3d_segmentation/README.md index 13714e8c136..e901cfaa33d 100644 --- a/tools/pot/openvino/tools/pot/api/samples/3d_segmentation/README.md +++ b/tools/pot/openvino/tools/pot/api/samples/3d_segmentation/README.md @@ -1,35 +1,29 @@ -# API usage sample for segmentation task {#pot_sample_3d_segmentation_README} +# Quantizatiing 3D Segmentation Model {#pot_example_3d_segmentation_README} -This sample demonstrates the use of the [Post-training Optimization Tool API](@ref pot_compression_api_README) for the task of quantizing a 3D segmentation model. +This example demonstrates the use of the [Post-training Optimization Tool API](@ref pot_compression_api_README) for the task of quantizing a 3D segmentation model. The [Brain Tumor Segmentation](https://github.com/openvinotoolkit/open_model_zoo/blob/master/models/public/brain-tumor-segmentation-0002/brain-tumor-segmentation-0002.md) model from PyTorch* is used for this purpose. A custom `DataLoader` is created to load images in NIfTI format from [Medical Segmentation Decathlon BRATS 2017](http://medicaldecathlon.com/) dataset for 3D semantic segmentation task -and the implementation of Dice Index metric is used for the model evaluation. In addition, this sample demonstrates how one can use image metadata obtained during image reading and -preprocessing to post-process the model raw output. +and the implementation of Dice Index metric is used for the model evaluation. In addition, this example demonstrates how one can use image metadata obtained during image reading and +preprocessing to post-process the model raw output. The code of the example is available on [GitHub](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/3d_segmentation). ## How to prepare the data -To run this sample, you will need to download the Brain Tumors 2017 part of the Medical Segmentation Decathlon image database http://medicaldecathlon.com/. +To run this example, you will need to download the Brain Tumors 2017 part of the Medical Segmentation Decathlon image database http://medicaldecathlon.com/. 3D MRI data in NIfTI format can be found in the `imagesTr` folder, and segmentation masks are in `labelsTr`. -## How to Run the Sample -In the instructions below, the Post-Training Optimization Tool directory `` is referred to: -- `/lib/python/site-packages/` in the case of PyPI installation, where `` is a Python* - environment where OpenVINO is installed and `` is a Python* version, for example `3.6`. +## How to Run the example - `` is the directory where Intel® Distribution of OpenVINO™ toolkit is installed. - -1. To get started, follow the [Installation Guide](@ref pot_InstallationGuide). -2. Launch [Model Downloader](@ref omz_tools_downloader) tool to download `brain-tumor-segmentation-0002` model from the Open Model Zoo repository. +1. Launch [Model Downloader](@ref omz_tools_downloader) tool to download `brain-tumor-segmentation-0002` model from the Open Model Zoo repository. ```sh - python3 ./downloader.py --name brain-tumor-segmentation-0002 + omz_downloader --name brain-tumor-segmentation-0002 ``` -3. Launch [Model Converter](@ref omz_tools_downloader) tool to generate Intermediate Representation (IR) files for the model: +2. Launch [Model Converter](@ref omz_tools_downloader) tool to generate Intermediate Representation (IR) files for the model: ```sh - python3 ./converter.py --name brain-tumor-segmentation-0002 --mo /mo.py + omz_converter --name brain-tumor-segmentation-0002 ``` -4. Launch the sample script: +3. Launch the example script from the example directory: ```sh - python3 /api/samples/3d_segmentation/3d_segmentation_sample.py -m -d --mask-dir + python3 ./3d_segmentation_example.py -m -d --mask-dir ``` Optional: you can specify .bin file of IR directly using the `-w`, `--weights` options. diff --git a/tools/pot/openvino/tools/pot/api/samples/README.md b/tools/pot/openvino/tools/pot/api/samples/README.md index d031e9c71e6..f77a267e6c6 100644 --- a/tools/pot/openvino/tools/pot/api/samples/README.md +++ b/tools/pot/openvino/tools/pot/api/samples/README.md @@ -1,4 +1,4 @@ -# Post-training Optimization Tool API samples {#pot_sample_README} +# Post-training Optimization Tool API Examples {#pot_example_README} @sphinxdirective @@ -6,53 +6,46 @@ :maxdepth: 1 :hidden: - Image Classification Quantization Sample - Accuracy-Aware Quantization Sample - Cascaded Model Quantization Sample - Semantic segmentation quantization sample - 3D Segmentation quantization sample - GNA speech sample + Quantizatiing Image Classification Model + Quantizatiing Object Detection Model with Accuracy Control + Quantizatiing Cascaded Model + Quantizatiing Semantic Segmentation Model + Quantizatiing 3D Segmentation Model + Quantizatiing for GNA Device @endsphinxdirective -The Post-training Optimization Tool contains multiple samples that demonstrate how to use its [Software API](@ref pot_compression_api_README) -to optimize DL models which require special inference pipeline, data loading or metric calculation that -are not supported through the `AccuracyCheker` or `Simplified` engines (see [Best Practices](../../../../../docs/BestPractices.md) for more details). +The Post-training Optimization Tool contains multiple examples that demonstrate how to use its [API](@ref pot_compression_api_README) +to optimize DL models. All available examples can be found on [GitHub](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples). -All available samples can be found in `/api/samples` folder, where `` is a directory where the Post-Training Optimization Tool is installed. -> **NOTE**: - `` is referred to `/lib/python/site-packages/` in the case of PyPI installation, where `` is a Python* -> environment where OpenVINO is installed and `` is a Python* version, for example `3.6`. -> `` is the directory where Intel® Distribution of OpenVINO™ toolkit is installed. +The following examples demonstrate the implementation of `Engine`, `Metric`, and `DataLoader` interfaces for various use cases: -There are currently the following samples that demonstrate the implementation of `Engine`, `Metric` and `DataLoader` interfaces -for classification, detection and segmentation tasks: - -1. [Classification sample](./classification/README.md) +1. [Quantizing Image Classification model](./classification/README.md) - Uses single `MobilenetV2` model from TensorFlow* - Implements `DataLoader` to load .JPEG images and annotations of Imagenet database - Implements `Metric` interface to calculate Accuracy at top-1 metric - Uses DefaultQuantization algorithm for quantization model -2. [Object Detection sample](./object_detection/README.md) +2. [Quantizing Object Detection Model with Accuracy Control](./object_detection/README.md) - Uses single `MobileNetV1 FPN` model from TensorFlow* - Implements `Dataloader` to load images of COCO database - Implements `Metric` interface to calculate mAP@[.5:.95] metric - Uses `AccuracyAwareQuantization` algorithm for quantization model -3. [Segmentation sample](./segmentation/README.md) +3. [Quantizing Semantic Segmentation Model](./segmentation/README.md) - Uses single `DeepLabV3` model from TensorFlow* - Implements `DataLoader` to load .JPEG images and annotations of Pascal VOC 2012 database - Implements `Metric` interface to calculate Mean Intersection Over Union metric - Uses DefaultQuantization algorithm for quantization model -4. [3D Segmentation sample](./3d_segmentation/README.md) +4. [Quantizing 3D Segmentation Model](./3d_segmentation/README.md) - Uses single `Brain Tumor Segmentation` model from PyTorch* - Implements `DataLoader` to load images in NIfTI format from Medical Segmentation Decathlon BRATS 2017 database - Implements `Metric` interface to calculate Dice Index metric - Demonstrates how to use image metadata obtained during data loading to post-process the raw model output - Uses DefaultQuantization algorithm for quantization model -5. [Face Detection sample](./face_detection/README.md) +5. [Quantizing Cascaded model](./face_detection/README.md) - Uses cascaded (composite) `MTCNN` model from Caffe* that consists of three separate models in an OpenVino™ Intermediate Representation (IR) - Implements `Dataloader` to load .jpg images of WIDER FACE database - Implements `Metric` interface to calculate Recall metric @@ -61,9 +54,9 @@ for classification, detection and segmentation tasks: OpenVino™ Inference Engine and process raw model output for the correct statistics collection - Uses DefaultQuantization algorithm for quantization model -6. [GNA speech sample](./speech/README.md) +6. [Quantizing for GNA Device](./speech/README.md) - Uses models from Kaldi* - Implements `DataLoader` to data in .ark format - Uses DefaultQuantization algorithm for quantization model -After execution of each sample above the quantized model is placed into the folder `optimized`. The accuracy validation of the quantized model is performed right after the quantization. +After execution of each example above the quantized model is placed into the folder `optimized`. The accuracy validation of the quantized model is performed right after the quantization. diff --git a/tools/pot/openvino/tools/pot/api/samples/classification/README.md b/tools/pot/openvino/tools/pot/api/samples/classification/README.md index 1de72eaec85..0bb60fc7adc 100644 --- a/tools/pot/openvino/tools/pot/api/samples/classification/README.md +++ b/tools/pot/openvino/tools/pot/api/samples/classification/README.md @@ -1,32 +1,27 @@ -# API usage sample for classification task {#pot_sample_classification_README} +# Quantizing Image Classification Model {#pot_example_classification_README} -This sample demonstrates the use of the [Post-training Optimization Tool API](@ref pot_compression_api_README) for the task of quantizing a classification model. +This example demonstrates the use of the [Post-training Optimization Tool API](@ref pot_compression_api_README) for the task of quantizing a classification model. The [MobilenetV2](https://github.com/openvinotoolkit/open_model_zoo/blob/master/models/public/mobilenet-v2-1.0-224/mobilenet-v2-1.0-224.md) model from TensorFlow* is used for this purpose. -A custom `DataLoader` is created to load the [ImageNet](http://www.image-net.org/) classification dataset and the implementation of Accuracy at top-1 metric is used for the model evaluation. +A custom `DataLoader` is created to load the [ImageNet](http://www.image-net.org/) classification dataset and the implementation of Accuracy at top-1 metric is used for the model evaluation. The code of the example is available on [GitHub](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/classification). ## How to prepare the data -To run this sample, you need to [download](http://www.image-net.org/download-faq) the validation part of the ImageNet image database and place it in a separate folder, +To run this example, you need to [download](http://www.image-net.org/download-faq) the validation part of the ImageNet image database and place it in a separate folder, which will be later referred as ``. Annotations to images should be stored in a separate .txt file (``) in the format `image_name label`. -## How to Run the Sample -In the instructions below, the Post-Training Optimization Tool directory `` is referred to: -- `/lib/python/site-packages/` in the case of PyPI installation, where `` is a Python* - environment where OpenVINO is installed and `` is a Python* version, for example `3.6`. - `` is the directory where Intel® Distribution of OpenVINO™ toolkit is installed. +## How to Run the example -1. To get started, follow the [Installation Guide](@ref pot_InstallationGuide). -2. Launch [Model Downloader](@ref omz_tools_downloader) tool to download `mobilenet-v2-1.0-224` model from the Open Model Zoo repository. +1. Launch [Model Downloader](@ref omz_tools_downloader) tool to download `mobilenet-v2-1.0-224` model from the Open Model Zoo repository. ```sh - python3 ./downloader.py --name mobilenet-v2-1.0-224 + omz_downloader --name mobilenet-v2-1.0-224 ``` -3. Launch [Model Converter](@ref omz_tools_downloader) tool to generate Intermediate Representation (IR) files for the model: +2. Launch [Model Converter](@ref omz_tools_downloader) tool to generate Intermediate Representation (IR) files for the model: ```sh - python3 ./converter.py --name mobilenet-v2-1.0-224 --mo /mo.py + omz_converter --name mobilenet-v2-1.0-224 --mo /mo.py ``` -4. Launch the sample script: +3. Launch the example script from the example directory: ```sh - python3 /api/samples/classification/classification_sample.py -m -a -d + python3 ./classification_example.py -m -a -d ``` Optional: you can specify .bin file of IR directly using the `-w`, `--weights` options. diff --git a/tools/pot/openvino/tools/pot/api/samples/face_detection/README.md b/tools/pot/openvino/tools/pot/api/samples/face_detection/README.md index b898fc0e2aa..69201af3668 100644 --- a/tools/pot/openvino/tools/pot/api/samples/face_detection/README.md +++ b/tools/pot/openvino/tools/pot/api/samples/face_detection/README.md @@ -1,37 +1,32 @@ -# API usage sample for face detection task {#pot_sample_face_detection_README} +# Quantizing Cascaded Face detection Model {#pot_example_face_detection_README} -This sample demonstrates the use of the [Post-training Optimization Tool API](@ref pot_compression_api_README) for the task of quantizing a face detection model. +This example demonstrates the use of the [Post-training Optimization Tool API](@ref pot_compression_api_README) for the task of quantizing a face detection model. The [MTCNN](https://github.com/openvinotoolkit/open_model_zoo/blob/master/models/public/mtcnn/mtcnn.md) model from Caffe* is used for this purpose. A custom `DataLoader` is created to load [WIDER FACE](http://shuoyang1213.me/WIDERFACE/) dataset for a face detection task -and the implementation of Recall metric is used for the model evaluation. In addition, this sample demonstrates how one can implement +and the implementation of Recall metric is used for the model evaluation. In addition, this example demonstrates how one can implement an engine to infer a cascaded (composite) model that is represented by multiple submodels in an OpenVino™ Intermediate Representation (IR) -and has a complex staged inference pipeline. +and has a complex staged inference pipeline. The code of the example is available on [GitHub](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/face_detection). ## How to prepare the data -To run this sample, you need to download the validation part of the Wider Face dataset http://shuoyang1213.me/WIDERFACE/. +To run this example, you need to download the validation part of the Wider Face dataset http://shuoyang1213.me/WIDERFACE/. Images with faces divided into categories are placed in the `WIDER_val/images` folder. Annotations in .txt format containing the coordinates of the face bounding boxes of the validation part of the dataset can be downloaded separately and are located in the `wider_face_split/wider_face_val_bbx_gt.txt` file. -## How to Run the Sample -In the instructions below, the Post-Training Optimization Tool directory `` is referred to: -- `/lib/python/site-packages/` in the case of PyPI installation, where `` is a Python* - environment where OpenVINO is installed and `` is a Python* version, for example `3.6`. - `` is the directory where Intel® Distribution of OpenVINO™ toolkit is installed. +## How to Run the example -1. To get started, follow the [Installation Guide](@ref pot_InstallationGuide). -2. Launch [Model Downloader](@ref omz_tools_downloader) tool to download `mtcnn` model from the Open Model Zoo repository. +1. Launch [Model Downloader](@ref omz_tools_downloader) tool to download `mtcnn` model from the Open Model Zoo repository. ```sh - python3 ./downloader.py --name mtcnn* + omz_downloader --name mtcnn* ``` -3. Launch [Model Converter](@ref omz_tools_downloader) tool to generate Intermediate Representation (IR) files for the model: +2. Launch [Model Converter](@ref omz_tools_downloader) tool to generate Intermediate Representation (IR) files for the model: ```sh - python3 ./converter.py --name mtcnn* --mo /mo.py + omz_converter --name mtcnn* --mo /mo.py ``` -4. Launch the sample script: +3. Launch the example script from the example directory: ```sh - python3 /api/samples/face_detection/face_detection_sample.py -pm + python3 ./face_detection_example.py -pm -rm -om -d -a ``` Optional: you can specify .bin files of corresponding IRs directly using the `-pw/--pnet-weights`, `-rw/--rnet-weights` and `-ow/--onet-weights` options. diff --git a/tools/pot/openvino/tools/pot/api/samples/object_detection/README.md b/tools/pot/openvino/tools/pot/api/samples/object_detection/README.md index d7ef179942f..04c3738ae7b 100644 --- a/tools/pot/openvino/tools/pot/api/samples/object_detection/README.md +++ b/tools/pot/openvino/tools/pot/api/samples/object_detection/README.md @@ -1,31 +1,26 @@ -# API usage sample for object_detection {#pot_sample_object_detection_README} +# Quantizing Object Detection Model with Accuracy Control {#pot_example_object_detection_README} -This sample demonstrates the use of the [Post-training Optimization Toolkit API](@ref pot_compression_api_README) to - quantize an object detection model in the [accuracy-aware mode](@ref pot_compression_algorithms_quantization_accuracy_aware_README). +This example demonstrates the use of the [Post-training Optimization Toolkit API](@ref pot_compression_api_README) to + quantize an object detection model in the [accuracy-aware mode](@ref accuracy_aware_README). The [MobileNetV1 FPN](https://github.com/openvinotoolkit/open_model_zoo/blob/master/models/public/ssd_mobilenet_v1_fpn_coco/ssd_mobilenet_v1_fpn_coco.md) model from TensorFlow* for object detection task is used for this purpose. A custom `DataLoader` is created to load the [COCO](https://cocodataset.org/) dataset for object detection task -and the implementation of mAP COCO is used for the model evaluation. +and the implementation of mAP COCO is used for the model evaluation. The code of the example is available on [GitHub](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/object_detection). ## How to prepare the data -To run this sample, you will need to download the validation part of the [COCO](https://cocodataset.org/). The images should be placed in a separate folder, which will be later referred as `` and annotation file `instances_val2017.json` later referred as ``. -## How to Run the Sample -In the instructions below, the Post-Training Optimization Tool directory `` is referred to: -- `/lib/python/site-packages/` in the case of PyPI installation, where `` is a Python* - environment where OpenVINO is installed and `` is a Python* version, for example `3.6`. - `` is the directory where Intel® Distribution of OpenVINO™ toolkit is installed. +To run this example, you will need to download the validation part of the [COCO](https://cocodataset.org/). The images should be placed in a separate folder, which will be later referred to as `` and the annotation file `instances_val2017.json` later referred to as ``. +## How to Run the example -1. To get started, follow the [Installation Guide](@ref pot_InstallationGuide). -2. Launch [Model Downloader](@ref omz_tools_downloader) tool to download `ssd_mobilenet_v1_fpn_coco` model from the Open Model Zoo repository. +1. Launch [Model Downloader](@ref omz_tools_downloader) tool to download `ssd_mobilenet_v1_fpn_coco` model from the Open Model Zoo repository. ```sh - python3 ./downloader.py --name ssd_mobilenet_v1_fpn_coco -3. Launch [Model Converter](@ref omz_tools_downloader) tool to generate Intermediate Representation (IR) files for the model: + omz_downloader --name ssd_mobilenet_v1_fpn_coco +2. Launch [Model Converter](@ref omz_tools_downloader) tool to generate Intermediate Representation (IR) files for the model: ```sh - python3 ./converter.py --name ssd_mobilenet_v1_fpn_coco --mo /mo.py + omz_converter --name ssd_mobilenet_v1_fpn_coco --mo /mo.py ``` -4. Launch the sample script: +3. Launch the example script from the example directory: ```sh - python /api/samples/object_detection/object_detection_sample.py -m -d --annotation-path + python ./object_detection_example.py -m -d --annotation-path ``` * Optional: you can specify .bin file of IR directly using the `-w`, `--weights` options. diff --git a/tools/pot/openvino/tools/pot/api/samples/segmentation/README.md b/tools/pot/openvino/tools/pot/api/samples/segmentation/README.md index f7f1da5b03e..bda999f8a9f 100644 --- a/tools/pot/openvino/tools/pot/api/samples/segmentation/README.md +++ b/tools/pot/openvino/tools/pot/api/samples/segmentation/README.md @@ -1,34 +1,29 @@ -# API usage sample for segmentation task {#pot_sample_segmentation_README} +# Quantizing Semantic Segmentation Model {#pot_example_segmentation_README} -This sample demonstrates the use of the [Post-training Optimization Tool API](@ref pot_compression_api_README) for the task of quantizing a segmentation model. +This example demonstrates the use of the [Post-training Optimization Tool API](@ref pot_compression_api_README) for the task of quantizing a segmentation model. The [DeepLabV3](https://github.com/openvinotoolkit/open_model_zoo/blob/master/models/public/deeplabv3/deeplabv3.md) model from TensorFlow* is used for this purpose. A custom `DataLoader` is created to load the [Pascal VOC 2012](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/) dataset for semantic segmentation task -and the implementation of Mean Intersection Over Union metric is used for the model evaluation. +and the implementation of Mean Intersection Over Union metric is used for the model evaluation. The code of the example is available on [GitHub](https://github.com/openvinotoolkit/openvino/tree/master/tools/pot/openvino/tools/pot/api/samples/segmentation). ## How to prepare the data -To run this sample, you will need to download the validation part of the Pascal VOC 2012 image database http://host.robots.ox.ac.uk/pascal/VOC/voc2012/#data. +To run this example, you will need to download the validation part of the Pascal VOC 2012 image database http://host.robots.ox.ac.uk/pascal/VOC/voc2012/#data. Images are placed in the `JPEGImages` folder, ImageSet file with the list of image names for the segmentation task can be found at `ImageSets/Segmentation/val.txt` and segmentation masks are kept in the `SegmentationClass` directory. -## How to Run the Sample -In the instructions below, the Post-Training Optimization Tool directory `` is referred to: -- `/lib/python/site-packages/` in the case of PyPI installation, where `` is a Python* - environment where OpenVINO is installed and `` is a Python* version, for example `3.6`. - `` is the directory where Intel® Distribution of OpenVINO™ toolkit is installed. +## How to Run the example -1. To get started, follow the [Installation Guide](@ref pot_InstallationGuide). -2. Launch [Model Downloader](@ref omz_tools_downloader) tool to download `deeplabv3` model from the Open Model Zoo repository. +1. Launch [Model Downloader](@ref omz_tools_downloader) tool to download `deeplabv3` model from the Open Model Zoo repository. ```sh - python3 ./downloader.py --name deeplabv3 + omz_downloader --name deeplabv3 ``` -3. Launch [Model Converter](@ref omz_tools_downloader) tool to generate Intermediate Representation (IR) files for the model: +2. Launch [Model Converter](@ref omz_tools_downloader) tool to generate Intermediate Representation (IR) files for the model: ```sh - python3 ./converter.py --name deeplabv3 --mo /mo.py + omz_converter --name deeplabv3 --mo /mo.py ``` -4. Launch the sample script: +3. Launch the example script from the example directory: ```sh - python3 /api/samples/segmentation/segmentation_sample.py -m -d --imageset-file --mask-dir + python3 ./segmentation_example.py -m -d --imageset-file --mask-dir ``` Optional: you can specify .bin file of IR directly using the `-w`, `--weights` options. diff --git a/tools/pot/openvino/tools/pot/api/samples/speech/README.md b/tools/pot/openvino/tools/pot/api/samples/speech/README.md index 9110e772e8f..1f237e97974 100644 --- a/tools/pot/openvino/tools/pot/api/samples/speech/README.md +++ b/tools/pot/openvino/tools/pot/api/samples/speech/README.md @@ -1,30 +1,25 @@ -# API usage sample for speech task on GNA {#pot_sample_speech_README} +# Quantizing for GNA Device {#pot_example_speech_README} -This sample demonstrates the use of the [Post-training Optimization Tool API](@ref pot_compression_api_README) for the task of quantizing a speech model for [GNA](@ref openvino_docs_OV_UG_supported_plugins_GNA) device. +This example demonstrates the use of the [Post-training Optimization Tool API](@ref pot_compression_api_README) for the task of quantizing a speech model for [GNA](@ref openvino_docs_OV_UG_supported_plugins_GNA) device. Quantization for GNA is different from CPU quantization due to device specific: GNA supports quantized inputs in INT16 and INT32 (for activations) precision and quantized weights in INT8 and INT16 precision. -This sample contains pre-selected quantization options based on the DefaultQuantization algorithm and created for models from [Kaldi](http://kaldi-asr.org/doc/) framework, and its data format. -A custom `ArkDataLoader` is created to load the dataset from files with .ark extension for speech analysis task. +This example contains pre-selected quantization options based on the DefaultQuantization algorithm and created for models from [Kaldi](http://kaldi-asr.org/doc/) framework, and its data format. +A custom `ArkDataLoader` is created to load the dataset from files with .ark extension for speech analysis task. ## How to prepare the data -To run this sample, you will need to use the .ark files for each model input from your ``. +To run this example, you will need to use the .ark files for each model input from your ``. For generating data from original formats to .ark, please, follow the [Kaldi data preparation tutorial](https://kaldi-asr.org/doc/data_prep.html). -## How to Run the Sample -In the instructions below, the Post-Training Optimization Tool directory `` is referred to: -- `/lib/python/site-packages/` in the case of PyPI installation, where `` is a Python* - environment where OpenVINO is installed and `` is a Python* version, for example `3.6`. - `` is the directory where Intel® Distribution of OpenVINO™ toolkit is installed. +## How to Run the example -1. To get started, follow the [Installation Guide](@ref pot_InstallationGuide). -2. Launch [Model Optimizer](@ref openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide) with the necessary options (for details follow the [instructions for Kaldi](@ref openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Kaldi) to generate Intermediate Representation (IR) files for the model: +1. Launch [Model Optimizer](@ref openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide) with the necessary options (for details follow the [instructions for Kaldi](@ref openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Kaldi) to generate Intermediate Representation (IR) files for the model: ```sh - python3 /mo.py --input_model [MODEL_OPTIMIZER_OPTIONS] + mo --input_model [MODEL_OPTIMIZER_OPTIONS] ``` -3. Launch the sample script: +2. Launch the example script: ```sh - python3 /api/samples/speech/gna_sample.py -m -w -d --input_names [LIST_OF_MODEL_INPUTS] --files_for_input [LIST_OF_INPUT_FILES] + python3 /api/examples/speech/gna_example.py -m -w -d --input_names [LIST_OF_MODEL_INPUTS] --files_for_input [LIST_OF_INPUT_FILES] ``` Required parameters: - `-i`, `--input_names` option. Defines list of model inputs; @@ -35,4 +30,4 @@ In the instructions below, the Post-Training Optimization Tool directory ` Date: Wed, 30 Mar 2022 16:43:32 +0300 Subject: [PATCH 02/39] [GPU] Add fusion through feature (#9674) * [GPU] Add fuse through feature * Apply review comments Tasks marked as not passed: * https://dev.azure.com/openvinoci/dldt/_build/results?buildId=313769&view=results * https://dev.azure.com/openvinoci/dldt/_build/results?buildId=313766&view=results --- .../include/intel_gpu/graph/program.hpp | 11 +- .../prepare_primitive_fusing.cpp | 18 +- .../prepare_primitive_fusing_through.cpp | 174 ++++++++++++++++++ .../src/graph/include/pass_manager.h | 6 + .../src/graph/include/program_node.h | 4 +- src/plugins/intel_gpu/src/graph/program.cpp | 33 +++- .../intel_gpu/src/graph/program_node.cpp | 12 +- src/plugins/intel_gpu/src/graph/softmax.cpp | 8 +- .../softmax_items_class_kernel_base.cpp | 21 ++- .../softmax/softmax_items_class_kernel_base.h | 9 + .../softmax/softmax_kernel_base.h | 6 + .../softmax/softmax_kernel_bf.cpp | 26 ++- .../softmax/softmax_kernel_bf.h | 8 +- .../softmax/softmax_kernel_fb.cpp | 23 ++- .../softmax/softmax_kernel_fb.h | 6 +- .../core/cl_kernels/softmax_gpu_bf.cl | 36 +++- .../core/cl_kernels/softmax_gpu_fb.cl | 40 +++- .../softmax_gpu_items_class_optimized.cl | 40 ++-- .../core/cl_kernels/softmax_gpu_ref.cl | 23 ++- .../kernel_selector/core/common/jitter.cpp | 1 - .../tests/fusions/convolution_fusion_test.cpp | 42 ++++- .../tests/fusions/fusion_test_common.hpp | 18 ++ .../tests/fusions/resample_fusion_test.cpp | 83 ++++++++- .../tests/fusions/softmax_fusion_test.cpp | 171 +++++++++++++++++ 24 files changed, 748 insertions(+), 71 deletions(-) create mode 100644 src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing_through.cpp create mode 100644 src/plugins/intel_gpu/tests/fusions/softmax_fusion_test.cpp diff --git a/src/plugins/intel_gpu/include/intel_gpu/graph/program.hpp b/src/plugins/intel_gpu/include/intel_gpu/graph/program.hpp index 7c5cba3a587..f9680daf73f 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/graph/program.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/graph/program.hpp @@ -44,8 +44,9 @@ struct program { friend class prepare_conv_eltw_fusing; // to be removed when possible friend class reorder_inputs; // to be removed when possible friend class remove_redundant_reorders; // to be removed when possible - friend class program_wrapper; // this class is intended to extend the interface of program for + friend class program_wrapper; // this class is intended to extend the interface of program for // the usage within tests_core_internal project only + friend class prepare_primitive_fusing_through; // to be removed when possible public: struct nodes_ordering { public: @@ -189,6 +190,12 @@ public: // returns if 'node' has been extracted and removed successfully bool extract_and_remove(program_node& node); + bool extract(program_node& node); + + bool move_node(program_node& node, + program_node& new_prev, + program_node& new_next); + // Fuses two nodes into fused_node and removes peer_node from graph void fuse_nodes(program_node& fused_node, program_node& peer_node, @@ -324,7 +331,7 @@ private: void rename(program_node& node, primitive_id const& new_id); void swap_names(program_node& node1, program_node& node2); - void replace_all_usages(program_node& old_node, program_node& new_node); + void replace_all_usages(program_node& old_node, program_node& new_node, bool remove_if_dangling = true); // old_node - node which will be replaced // new_node - node which will replace the old one diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp index 0bd7ac0e776..bb26cbd4125 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp @@ -741,10 +741,7 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { should_fuse |= input_data.is_type(); - // Here we need to check that Eltwise already has fused ops to avoid missing Activation primitive in - // case `Conv -> Eltwise -> Activation` which will be replaced via fused_conv_eltwise primitive later - // without handling any fused ops - should_fuse |= input_data.is_type() && eltwise_supports_fusings(input_data.as()) && input_data.has_fused_primitives(); + should_fuse |= input_data.is_type() && eltwise_supports_fusings(input_data.as()); if (!should_fuse) return; @@ -832,6 +829,14 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { auto out_dt_is_i8_u8 = data_type_traits::is_i8_u8(out_dt); auto in_dt_is_i8_u8 = data_type_traits::is_i8_u8(in_dt); + bool per_tensor_values = quantize_node.get_scale_shift_opt() && + quantize_node.get_per_tensor_input_scale() && + quantize_node.get_per_tensor_input_shift() && + quantize_node.get_per_tensor_input_range() && + quantize_node.get_per_tensor_output_scale() && + quantize_node.get_per_tensor_output_shift() && + quantize_node.get_per_tensor_output_range(); + bool should_fuse = input_data.is_type() && ((out_dt == data_types::bin && quantize_node.get_dependencies().size() == 5 && @@ -911,6 +916,11 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { should_fuse |= input_data.is_type() && quantize_node.get_scale_shift_opt(); + should_fuse |= input_data.is_type() && + input_data.as().get_primitive()->dimension == softmax::dimension_t::normalize_f && + per_tensor_values; + + if (!should_fuse) return; diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing_through.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing_through.cpp new file mode 100644 index 00000000000..9026b42ca33 --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing_through.cpp @@ -0,0 +1,174 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +#include "intel_gpu/runtime/error_handler.hpp" +#include "pass_manager.h" +#include "program_helpers.h" +#include "strided_slice_inst.h" +#include "reshape_inst.h" +#include "data_inst.h" +#include "eltwise_inst.h" +#include "mutable_data_inst.h" +#include +#include + +using namespace cldnn; + +void prepare_primitive_fusing_through::run(program& p) { + auto try_fuse_through = [&](program_node& node) -> std::vector { + // This function tries to fuse peer_node to first non reorder or reshape previous primitive. + // It returns chain of primitives (reshapes and reorders) including potential fused_node (e.g. Conv, FC, etc) + // at the end of vector. + // There are some limitations: + // 1. As for now we can fuse only through chain of reorders and reshapes primitives + // 2. There are three types of supported primitives: + // - Quantize with per-tensor parameters + // - Eltwise with single scale value in constant buffer + // - Activation w/o additional buffers + // 3. Paddings are not allowed for fused node + auto can_raise_up_through = [](program_node* node) { + if (!node->is_type() && !node->is_type()) + return false; + + if (node->get_dependencies().empty()) + return false; + + if (node->get_users().size() > 1) + return false; + + if (node->is_type() && + node->get_output_layout().data_type != node->get_dependency(0).get_output_layout().data_type) + return false; + + return true; + }; + + std::vector pass_through; + program_node* fuse_through = &node; + pass_through.push_back(fuse_through); + + bool can_raise_up = can_raise_up_through(fuse_through); + while (can_raise_up) { + fuse_through = &fuse_through->get_dependency(0); + can_raise_up = can_raise_up_through(fuse_through); + pass_through.push_back(fuse_through); + } + + return pass_through; + }; + + auto node_itr = p.get_processing_order().begin(); + while (node_itr != p.get_processing_order().end()) { + auto node = (*node_itr++); + cldnn::program_node* input_node; + + if (node->is_output() || node->is_constant()) + continue; + + if (node->is_type()) { + if (node->get_dependencies().size() > 1) + continue; + + input_node = &node->get_dependency(0); + } else if (node->is_type()) { + auto& quantize_node = node->as(); + bool per_tensor_values = quantize_node.get_scale_shift_opt() && + quantize_node.get_per_tensor_input_scale() && + quantize_node.get_per_tensor_input_shift() && + quantize_node.get_per_tensor_input_range() && + quantize_node.get_per_tensor_output_scale() && + quantize_node.get_per_tensor_output_shift() && + quantize_node.get_per_tensor_output_range(); + + if (!per_tensor_values) + continue; + + input_node = &node->get_dependency(0); + } else if (node->is_type()) { + if (node->get_dependencies().size() !=2) + continue; + + size_t second_input_idx = 0; + bool has_constant_input = false; + for (size_t i = 0; i < node->get_dependencies().size(); i++) { + auto& dep = node->get_dependency(i); + if (dep.is_constant() && dep.get_output_layout().size == cldnn::tensor(1)) { + second_input_idx = i ^ 1; + has_constant_input = true; + break; + } + } + + if (!has_constant_input) + continue; + + input_node = &node->get_dependency(second_input_idx); + } else { + continue; + } + + auto fuse_through_order = try_fuse_through(*input_node); + bool use_fuse_through = fuse_through_order.size() > 1; + bool fused_node_has_multiple_users = fuse_through_order.back()->get_users().size() > 1; + + if (!use_fuse_through || fused_node_has_multiple_users) + continue; + + if (static_cast(node->get_output_layout().data_padding)) + continue; + + auto new_prev = fuse_through_order[fuse_through_order.size() - 1]; + auto new_next = fuse_through_order[fuse_through_order.size() - 2]; + + if (new_prev->is_type() || + new_prev->is_type() || + new_prev->is_type()) + continue; + + std::vector dependencies; + for (auto dep : node->get_dependencies()) { + if (dep == input_node) + continue; + dependencies.push_back(dep); + } + + for (auto dep : dependencies) + p.remove_connection(*dep, *node); + + p.move_node(*node, *new_prev, *new_next); + + for (auto dep : dependencies) + p.add_connection(*dep, *node); + + // Update node's layout and keep original data type + auto target_dt = node->get_output_layout().data_type; + layout new_node_layout = new_prev->get_output_layout(); + new_node_layout.data_type = target_dt; + node->set_output_layout(new_node_layout, false); + + // Update intermediate nodes + auto node_itr = std::next(fuse_through_order.rbegin()); + while (node_itr != fuse_through_order.rend()) { + auto itermediate_node = *node_itr++; + + if (itermediate_node->is_type()) { + // We can't modify reorder's output type directly, so replace old node with new one + auto reorder_layout = itermediate_node->get_output_layout(); + reorder_layout.data_type = target_dt; + auto r_prim = std::make_shared(itermediate_node->id() + "_reorder_to_req_dt", itermediate_node->id(), reorder_layout); + auto& new_reorder = p.get_or_create(r_prim); + + p.add_intermediate(new_reorder, *itermediate_node->get_users().front(), *itermediate_node); + new_reorder.recalc_output_layout(false); + + p.add_optimized_primitive_info(itermediate_node->id()); + p.extract_and_remove(*itermediate_node); + } else { + itermediate_node->recalc_output_layout(false); + } + } + } +} diff --git a/src/plugins/intel_gpu/src/graph/include/pass_manager.h b/src/plugins/intel_gpu/src/graph/include/pass_manager.h index 7812dc481c0..804d65a9757 100644 --- a/src/plugins/intel_gpu/src/graph/include/pass_manager.h +++ b/src/plugins/intel_gpu/src/graph/include/pass_manager.h @@ -185,6 +185,12 @@ private: void conv_eltwise_read_write_opt(program& p, program_node* node); }; +class prepare_primitive_fusing_through : public base_pass { +public: + prepare_primitive_fusing_through() : base_pass("prepare_primitive_fusing_through") {} + void run(program& p) override; +}; + class prepare_primitive_fusing : public base_pass { public: explicit prepare_primitive_fusing(layout_optimizer& lo_ref) : diff --git a/src/plugins/intel_gpu/src/graph/include/program_node.h b/src/plugins/intel_gpu/src/graph/include/program_node.h index a8380c867bc..b6160dec5e1 100644 --- a/src/plugins/intel_gpu/src/graph/include/program_node.h +++ b/src/plugins/intel_gpu/src/graph/include/program_node.h @@ -159,10 +159,10 @@ public: program_node& get_dependency(size_t idx) const { return *dependencies.at(idx); } // replaces idx-th dependency of 'this' with 'new_dep', calls program::remove_if_dangling(old_dep) - void replace_dependency(size_t idx, program_node& new_dep); + void replace_dependency(size_t idx, program_node& new_dep, bool remove_if_dangling = true); // searches for 'old_dep' in dependencies list of 'this' and replaces it with 'new_dep', calls // program::remove_if_dangling(old_dep) - void replace_dependency(program_node const& old_dep, program_node& new_dep); + void replace_dependency(program_node const& old_dep, program_node& new_dep, bool remove_if_dangling = true); std::vector get_dependencies_ids() const; diff --git a/src/plugins/intel_gpu/src/graph/program.cpp b/src/plugins/intel_gpu/src/graph/program.cpp index 8822a150a6c..3151b02dc46 100644 --- a/src/plugins/intel_gpu/src/graph/program.cpp +++ b/src/plugins/intel_gpu/src/graph/program.cpp @@ -509,6 +509,8 @@ void program::pre_optimize_graph(bool is_internal) { reorder_factory rf; if (options.get()->enabled()) { + apply_opt_pass(); + apply_opt_pass(lo); apply_opt_pass(lo); @@ -877,13 +879,13 @@ void program::swap_names(program_node& node1, program_node& node2) { std::swap(_extract_id(node1), _extract_id(node2)); } -void program::replace_all_usages(program_node& old_node, program_node& new_node) { +void program::replace_all_usages(program_node& old_node, program_node& new_node, bool remove_if_dangling) { // We need a copy of users of old_node because old_node may be removed when doing replace_dependency() const std::list users(old_node.users); auto itr = users.begin(); while (itr != users.end()) { auto user = *(itr++); - user->replace_dependency(old_node, new_node); + user->replace_dependency(old_node, new_node, remove_if_dangling); } } @@ -967,7 +969,7 @@ bool program::remove_if_dangling(program_node& node) { return true; } -bool program::extract_and_remove(program_node& node) { +bool program::extract(program_node& node) { if (node.get_dependencies().size() != 1) return false; @@ -1006,13 +1008,32 @@ bool program::extract_and_remove(program_node& node) { node.dependencies.clear(); if (!node.is_endpoint()) - replace_all_usages(node, input); - else - remove_if_dangling(node); + replace_all_usages(node, input, false); + + if (std::find(processing_order.begin(), processing_order.end(), &node) != processing_order.end()) + processing_order.erase(&node); return true; } +bool program::extract_and_remove(program_node& node) { + if (extract(node)) { + return remove_if_dangling(node); + } + + return false; +} + +bool program::move_node(program_node& node, + program_node& new_prev, + program_node& new_next) { + if (extract(node)) { + add_intermediate(node, new_next, new_prev); + return true; + } + + return false; +} void program::fuse_nodes(program_node &fused_node, program_node &peer_node, diff --git a/src/plugins/intel_gpu/src/graph/program_node.cpp b/src/plugins/intel_gpu/src/graph/program_node.cpp index 7fb88a67469..189afacb917 100644 --- a/src/plugins/intel_gpu/src/graph/program_node.cpp +++ b/src/plugins/intel_gpu/src/graph/program_node.cpp @@ -32,7 +32,7 @@ program_node::program_node(std::shared_ptr prim, program& prog) output_layout.data_padding = prim->output_padding; } -void program_node::replace_dependency(size_t idx, program_node& new_dep) { +void program_node::replace_dependency(size_t idx, program_node& new_dep, bool remove_if_dangling) { if (idx >= dependencies.size()) return; if (dependencies[idx] == &new_dep) @@ -47,16 +47,18 @@ void program_node::replace_dependency(size_t idx, program_node& new_dep) { if (it != dependencies[idx]->users.end()) { dependencies[idx]->users.erase(it); } - myprog.remove_if_dangling(*dependencies[idx]); + + if (remove_if_dangling) + myprog.remove_if_dangling(*dependencies[idx]); dependencies[idx] = &new_dep; new_dep.users.push_back(this); } -void program_node::replace_dependency(program_node const& old_dep, program_node& new_dep) { +void program_node::replace_dependency(program_node const& old_dep, program_node& new_dep, bool remove_if_dangling) { for (size_t i = 0; i < dependencies.size(); ++i) if (dependencies[i] == &old_dep) - return replace_dependency(i, new_dep); + return replace_dependency(i, new_dep, remove_if_dangling); } std::vector program_node::get_dependencies_ids() const { @@ -104,10 +106,10 @@ std::unique_ptr program_node::desc_to_json() const { node_info->add("output layout", output_layout_info); - node_info->add("in data flow", bool_to_str(data_flow)); node_info->add("constant", bool_to_str(constant)); node_info->add("in data flow", bool_to_str(data_flow)); node_info->add("output", bool_to_str(output)); + node_info->add("optimized", bool_to_str(optimized)); json_composite fused_nodes_info; size_t index = 0; diff --git a/src/plugins/intel_gpu/src/graph/softmax.cpp b/src/plugins/intel_gpu/src/graph/softmax.cpp index 48df613d96e..3dfdd293b26 100644 --- a/src/plugins/intel_gpu/src/graph/softmax.cpp +++ b/src/plugins/intel_gpu/src/graph/softmax.cpp @@ -16,7 +16,13 @@ primitive_type_id softmax::type_id() { layout softmax_inst::calc_output_layout(softmax_node const& node) { assert(static_cast(node.get_primitive()->output_data_type) == false && "Output data type forcing is not supported for softmax_node!"); - return node.input().get_output_layout(); + + auto output_layout = node.input().get_output_layout(); + + if (node.has_fused_primitives()) + output_layout.data_type = node.get_fused_output_layout().data_type; + + return output_layout; } std::string softmax_inst::to_string(softmax_node const& node) { diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_items_class_kernel_base.cpp b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_items_class_kernel_base.cpp index d19f913a66e..45ceabbd9cb 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_items_class_kernel_base.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_items_class_kernel_base.cpp @@ -12,6 +12,8 @@ ParamsKey SoftmaxItemsClassKernelBase::GetDefaultSupportedKey() { k.EnableInputDataType(Datatype::F32); k.EnableOutputDataType(Datatype::F16); k.EnableOutputDataType(Datatype::F32); + k.EnableOutputDataType(Datatype::INT8); + k.EnableOutputDataType(Datatype::UINT8); k.EnableInputLayout(DataLayout::byxf); k.EnableInputLayout(DataLayout::bfyx); k.EnableInputLayout(DataLayout::yxfb); @@ -34,6 +36,7 @@ ParamsKey SoftmaxItemsClassKernelBase::GetDefaultSupportedKey() { k.EnableSoftmaxDim(SoftmaxDim::Y); k.EnableSoftmaxDim(SoftmaxDim::Z); k.EnableSoftmaxDim(SoftmaxDim::FEATURE); + k.EnableDifferentTypes(); k.EnableTensorOffset(); k.EnableTensorPitches(); k.EnableBatching(); @@ -58,6 +61,8 @@ std::vector SoftmaxItemsClassKernelBase::GetSoftmaxDimGlobalSizes(Softma JitConstants SoftmaxItemsClassKernelBase::GetJitConstants(const softmax_params& params, DispatchData dispatchData) const { auto jit = SoftmaxKernelBase::GetJitConstants(params, dispatchData); + std::vector idx_order; + const auto ndims = params.inputs[0].GetDims().size(); switch (params.dim) { case SoftmaxDim::X: jit.AddConstants({ @@ -72,6 +77,7 @@ JitConstants SoftmaxItemsClassKernelBase::GetJitConstants(const softmax_params& MakeJitConstant("OUTPUT_OTHER1_PITCH", "OUTPUT_FEATURE_PITCH"), MakeJitConstant("OUTPUT_CLASS_PITCH", "OUTPUT_X_PITCH"), }); + idx_order = {"batch", "other1", ndims == 5 ? "other2" : "0", "other0", "cls"}; break; case SoftmaxDim::Y: jit.AddConstants({ @@ -86,6 +92,7 @@ JitConstants SoftmaxItemsClassKernelBase::GetJitConstants(const softmax_params& MakeJitConstant("OUTPUT_OTHER2_PITCH", "OUTPUT_Z_PITCH"), MakeJitConstant("OUTPUT_CLASS_PITCH", "OUTPUT_Y_PITCH"), }); + idx_order = {"batch", "other1", ndims == 5 ? "other2" : "0", "cls", "other0"}; break; case SoftmaxDim::Z: jit.AddConstants({ @@ -100,6 +107,7 @@ JitConstants SoftmaxItemsClassKernelBase::GetJitConstants(const softmax_params& MakeJitConstant("OUTPUT_OTHER2_PITCH", "OUTPUT_Y_PITCH"), MakeJitConstant("OUTPUT_CLASS_PITCH", "OUTPUT_Z_PITCH"), }); + idx_order = {"batch", "other1", "cls", "other2", "other0"}; break; case SoftmaxDim::FEATURE: jit.AddConstants({ @@ -114,14 +122,21 @@ JitConstants SoftmaxItemsClassKernelBase::GetJitConstants(const softmax_params& MakeJitConstant("OUTPUT_OTHER2_PITCH", "OUTPUT_Z_PITCH"), MakeJitConstant("OUTPUT_CLASS_PITCH", "OUTPUT_FEATURE_PITCH"), }); + idx_order = {"batch", "cls", ndims == 5 ? "other2" : "0", "other1", "other0"}; break; default: break; } - // TODO: W/A - currently using low precision accumulator type. (for testing only) - if (params.outputs[0].GetDType() == Datatype::F16) { - jit.AddConstant(MakeJitConstant("ACCUMULATOR_TYPE", "half")); + auto acc_dt = GetAccumulatorType(params); + jit.Merge(MakeTypeJitConstants(acc_dt, "ACCUMULATOR")); + + if (!params.fused_ops.empty()) { + FusedOpsConfiguration conf = {"", + idx_order, + "res", + acc_dt}; + jit.Merge(MakeFusedOpsJitConstants(params, {conf})); } return jit; diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_items_class_kernel_base.h b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_items_class_kernel_base.h index 8ee53dbee5b..4f22fa2fece 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_items_class_kernel_base.h +++ b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_items_class_kernel_base.h @@ -17,5 +17,14 @@ protected: JitConstants GetJitConstants(const softmax_params& params, DispatchData dispatchData) const override; static ParamsKey GetDefaultSupportedKey(); static std::vector GetSoftmaxDimGlobalSizes(SoftmaxDim dim, const DataTensor& output); + Datatype GetAccumulatorType(const softmax_params& params) const { + if (params.inputs[0].GetDType() == Datatype::F16) + return Datatype::F16; + else + return Datatype::F32; + } + std::vector GetSupportedFusedOps() const override { + return { FusedOpType::QUANTIZE }; + } }; } // namespace kernel_selector diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_base.h b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_base.h index bd6515b180b..8ebbd06adce 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_base.h +++ b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_base.h @@ -52,6 +52,12 @@ protected: virtual JitConstants GetJitConstants(const softmax_params& params, DispatchData dispatchData) const; virtual DispatchData SetDefault(const softmax_params& params, const optional_params& optParams) const; KernelsData GetCommonKernelsData(const Params& params, const optional_params& optParams) const; + Datatype GetActivationType(const softmax_params& params) const { + if (params.inputs[0].GetDType() == Datatype::F16) + return Datatype::F16; + else + return Datatype::F32; + } }; class SoftmaxKernelBaseBF : public SoftmaxKernelBase { diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_bf.cpp b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_bf.cpp index 6c671acb3d9..ccebb8e4db4 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_bf.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_bf.cpp @@ -12,6 +12,8 @@ ParamsKey SoftmaxKernel_bf::GetSupportedKey() const { k.EnableInputDataType(Datatype::F32); k.EnableOutputDataType(Datatype::F16); k.EnableOutputDataType(Datatype::F32); + k.EnableOutputDataType(Datatype::UINT8); + k.EnableOutputDataType(Datatype::INT8); k.EnableInputLayout(DataLayout::bfyx); k.EnableInputLayout(DataLayout::bf); k.EnableOutputLayout(DataLayout::bfyx); @@ -19,6 +21,7 @@ ParamsKey SoftmaxKernel_bf::GetSupportedKey() const { k.EnableSoftmaxDim(SoftmaxDim::X); // in case that it can be flatten k.EnableSoftmaxDim(SoftmaxDim::Y); k.EnableSoftmaxDim(SoftmaxDim::FEATURE); + k.EnableDifferentTypes(); k.EnableBatching(); return k; } @@ -64,4 +67,25 @@ KernelsPriority SoftmaxKernel_bf::GetKernelsPriority(const Params& /*params*/, c KernelsData SoftmaxKernel_bf::GetKernelsData(const Params& params, const optional_params& optionalParams) const { return GetCommonKernelsData(params, optionalParams); } -} // namespace kernel_selector \ No newline at end of file + +JitConstants SoftmaxKernel_bf::GetJitConstants(const softmax_params& params, DispatchData dispatchData) const { + auto jit = Parent::GetJitConstants(params, dispatchData); + + jit.Merge(MakeTypeJitConstants(GetActivationType(params), "ACTIVATION")); + + if (!params.fused_ops.empty()) { + auto input_dt = GetActivationType(params); + FusedOpsConfiguration conf_main = {"_MAIN", + {"data_set_offset", "in_data_set_idx + i * workers_per_data_set", "0", "0"}, + "dequantized", + input_dt}; + FusedOpsConfiguration conf_leftovers = {"_LEFTOVERS", + {"data_set_offset", "workers_per_data_set * ITEMS_NUM + in_data_set_idx", "0", "0"}, + "dequantized", + input_dt}; + jit.Merge(MakeFusedOpsJitConstants(params, {conf_main, conf_leftovers})); + } + + return jit; +} +} // namespace kernel_selector diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_bf.h b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_bf.h index b5f366be103..953a9022db6 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_bf.h +++ b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_bf.h @@ -16,6 +16,12 @@ public: KernelsData GetKernelsData(const Params& params, const optional_params& options) const override; KernelsPriority GetKernelsPriority(const Params& params, const optional_params& options) const override; ParamsKey GetSupportedKey() const override; + +protected: DispatchData SetDefault(const softmax_params& params, const optional_params& optParams) const override; + JitConstants GetJitConstants(const softmax_params& params, DispatchData dispatchData) const override; + std::vector GetSupportedFusedOps() const override { + return { FusedOpType::QUANTIZE }; + } }; -} // namespace kernel_selector \ No newline at end of file +} // namespace kernel_selector diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_fb.cpp b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_fb.cpp index 5cf1a636393..c213d2f9831 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_fb.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_fb.cpp @@ -95,4 +95,25 @@ KernelsData SoftmaxKernel_fb::GetKernelsData(const Params& params, const optiona } return GetCommonKernelsData(params, optParams); } -} // namespace kernel_selector \ No newline at end of file + +JitConstants SoftmaxKernel_fb::GetJitConstants(const softmax_params& params, DispatchData dispatchData) const { + auto jit = Parent::GetJitConstants(params, dispatchData); + + jit.Merge(MakeTypeJitConstants(GetActivationType(params), "ACTIVATION")); + + if (!params.fused_ops.empty()) { + auto input_dt = GetActivationType(params); + FusedOpsConfiguration conf_main = {"_MAIN", + {"global_id", "LWS * i", "0", "0"}, + "dequantized", + input_dt}; + FusedOpsConfiguration conf_leftovers = {"_LEFTOVERS", + {"global_id", "LWS * ITEMS_NUM", "0", "0"}, + "dequantized", + input_dt}; + jit.Merge(MakeFusedOpsJitConstants(params, {conf_main, conf_leftovers})); + } + + return jit; +} +} // namespace kernel_selector diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_fb.h b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_fb.h index 8a0640993d4..46147f8a2e0 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_fb.h +++ b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/softmax/softmax_kernel_fb.h @@ -20,5 +20,9 @@ public: protected: bool Validate(const Params& p, const optional_params& o) const override; DispatchData SetDefault(const softmax_params& params, const optional_params& optParams) const override; + JitConstants GetJitConstants(const softmax_params& params, DispatchData dispatchData) const override; + std::vector GetSupportedFusedOps() const override { + return { FusedOpType::QUANTIZE }; + } }; -} // namespace kernel_selector \ No newline at end of file +} // namespace kernel_selector diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/cl_kernels/softmax_gpu_bf.cl b/src/plugins/intel_gpu/src/kernel_selector/core/cl_kernels/softmax_gpu_bf.cl index 48f8a0d4940..95ba114c5de 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/core/cl_kernels/softmax_gpu_bf.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/core/cl_kernels/softmax_gpu_bf.cl @@ -7,23 +7,28 @@ __attribute__((reqd_work_group_size(LWS, 1, 1))) -KERNEL (softmax_gpu_continoues_bfyx)(const __global UNIT_TYPE* input, __global UNIT_TYPE* output) -{ +KERNEL (softmax_gpu_continuous_bfyx)( + const __global INPUT0_TYPE* input, + __global OUTPUT_TYPE* output +#if HAS_FUSED_OPS_DECLS + , FUSED_OPS_DECLS +#endif +) { const uint data_set_idx = get_global_id(1); //in processing of which data set this WI participates? const uint workers_per_data_set = LWS; //how many WI participates in processing of one data set const uint in_data_set_idx = get_global_id(0); //this WI's id in group of items processing single data set const uint data_set_size = DATA_SET_SIZE; //how many elements are in one data set - const uint data_sets_count = DATA_SETS_COUNT; //how many data sets are in the processing payload + const uint data_sets_count = DATA_SETS_COUNT; //how many data sets are in the processing payload const uint data_set_offset = data_set_idx * data_set_size; const uint my_data_offset = data_set_offset + in_data_set_idx; - UNIT_TYPE my_chunk[ITEMS_NUM + 1]; - UNIT_TYPE my_maximum = -UNIT_VAL_MAX; - UNIT_TYPE my_sum = UNIT_VAL_ZERO; - UNIT_TYPE tmp; + INPUT0_TYPE my_chunk[ITEMS_NUM + 1]; + INPUT0_TYPE my_maximum = -UNIT_VAL_MAX; + INPUT0_TYPE my_sum = UNIT_VAL_ZERO; + INPUT0_TYPE tmp; - __local UNIT_TYPE lg_storage[LWS]; + __local INPUT0_TYPE lg_storage[LWS]; //each WI reads ITEMS_NUM consecutive items from batch for (uint i=0; i { public: - void execute(convolution_test_params& p) { + void execute(convolution_test_params& p, std::map> expected_fused_primitives_ids = {}) { auto input_prim = get_mem(get_input_layout(p)); network network_not_fused(this->engine, this->topology_non_fused, bo_not_fused); network network_fused(this->engine, this->topology_fused, bo_fused); @@ -123,6 +123,7 @@ public: network_not_fused.set_input_data("input", input_prim); compare(network_not_fused, network_fused, p, true); + check_fusions_correctness(network_fused, expected_fused_primitives_ids); } layout get_input_layout(convolution_test_params& p) { @@ -2641,6 +2642,38 @@ TEST_P(conv_fp32_reorder_bfyx_to_fsv32_conv_fused_activation, have_fused_activat auto dw_weights_layout = layout{ p.default_type, format::goiyx, dw_tensor }; ov::Strides dw_stride = { 1, 1 }; + create_topologies( + input_layout("input", get_input_layout(p)), + data("weights", get_mem(get_weights_layout(p), -127, 127)), + data("weights_dw", get_mem(dw_weights_layout, -127, 127)), + data("actv_params", get_mem(get_per_channel_layout(p), -127, 127)), + convolution("conv_prim", "input", { "weights" }, p.groups, p.stride, p.pad, p.dilation), + reorder("reorder_fsv32", "conv_prim", format::fs_b_yx_fsv32, data_types::f32), + activation("activation_quantize", "reorder_fsv32", "actv_params", activation_func::relu_negative_slope), + convolution("conv_prim2", "activation_quantize", { "weights_dw" }, p.out_shape.feature[0], dw_stride, p.pad, p.dilation), + activation("activation", "conv_prim2", activation_func::abs) + ); + + implementation_desc conv_impl = { format::fs_b_yx_fsv32, "" }; + bo_fused.set_option(build_option::force_implementations({ { "conv_prim2", conv_impl } })); + bo_fused.set_option(build_option::force_implementations({ { "activation", conv_impl } })); + + execute(p); +} + +INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_fp32_reorder_bfyx_to_fsv32_conv_fused_activation, ::testing::ValuesIn(std::vector{ + convolution_test_params{ FSV32_CASE_CONV_FP32_1, 5, 6 } +})); + +// activation will be fused through 'reorder_fsv32' and 'reorder_fsv32' will be fused as well +class conv_fp32_reorder_bfyx_to_fsv32_conv_fused_through_activation : public ConvReorderFusingTest {}; +TEST_P(conv_fp32_reorder_bfyx_to_fsv32_conv_fused_through_activation, have_fused_through_activation) { + auto p = GetParam(); + + auto dw_tensor = cldnn::tensor(group(p.out_shape.feature[0]), batch(1), feature(1), spatial(2, 2)); + auto dw_weights_layout = layout{ p.default_type, format::goiyx, dw_tensor }; + ov::Strides dw_stride = { 1, 1 }; + create_topologies( input_layout("input", get_input_layout(p)), data("weights", get_mem(get_weights_layout(p), -127, 127)), @@ -2656,10 +2689,11 @@ TEST_P(conv_fp32_reorder_bfyx_to_fsv32_conv_fused_activation, have_fused_activat bo_fused.set_option(build_option::force_implementations({ { "conv_prim2", conv_impl } })); bo_fused.set_option(build_option::force_implementations({ { "activation", conv_impl } })); - execute(p); + execute(p, {{"conv_prim", {"activation_quantize"}}}); } -INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_fp32_reorder_bfyx_to_fsv32_conv_fused_activation, ::testing::ValuesIn(std::vector{ - convolution_test_params{ FSV32_CASE_CONV_FP32_1, 5, 6 } + +INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_fp32_reorder_bfyx_to_fsv32_conv_fused_through_activation, ::testing::ValuesIn(std::vector{ + convolution_test_params{ FSV32_CASE_CONV_FP32_1, 4, 6 } })); // 'reorder_fsv32' is being fused even if it has 'padding' diff --git a/src/plugins/intel_gpu/tests/fusions/fusion_test_common.hpp b/src/plugins/intel_gpu/tests/fusions/fusion_test_common.hpp index 8ef703bdc14..04a2beb45df 100644 --- a/src/plugins/intel_gpu/tests/fusions/fusion_test_common.hpp +++ b/src/plugins/intel_gpu/tests/fusions/fusion_test_common.hpp @@ -94,6 +94,24 @@ public: } } + void check_fusions_correctness(network& network_fused, std::map> expected_fused_primitives_ids = {}) { + if (expected_fused_primitives_ids.size()) { + auto primitives_info = network_fused.get_primitives_info(); + for (auto& prim : expected_fused_primitives_ids) { + auto info = std::find_if(primitives_info.begin(), primitives_info.end(), + [&prim](const primitive_info& info) -> bool { return info.original_id == prim.first; }); + if (info != primitives_info.end()) { + auto fused_primitives = info->c_fused_ids; + for (auto& expected_fused_prim : prim.second) + if (std::find(fused_primitives.begin(), fused_primitives.end(), expected_fused_prim) == fused_primitives.end()) + FAIL() << "Couldn't find requested fused primitive id " + prim.first; + } else { + FAIL() << "Couldn't find requested primitive id " + prim.first; + } + } + } + } + cldnn::memory::ptr get_mem(cldnn::layout l) { auto prim = engine.allocate_memory(l); tensor s = l.size; diff --git a/src/plugins/intel_gpu/tests/fusions/resample_fusion_test.cpp b/src/plugins/intel_gpu/tests/fusions/resample_fusion_test.cpp index 27506403429..fe15b32ef5b 100644 --- a/src/plugins/intel_gpu/tests/fusions/resample_fusion_test.cpp +++ b/src/plugins/intel_gpu/tests/fusions/resample_fusion_test.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include @@ -32,7 +33,7 @@ struct resample_test_params { class ResamplePrimitiveFusingTest : public ::BaseFusingTest { public: - void execute(resample_test_params& p) { + void execute(resample_test_params& p, std::map> expected_fused_primitives_ids = {}) { auto input_prim = get_mem(get_input_layout(p)); network network_not_fused(this->engine, this->topology_non_fused, bo_not_fused); network network_fused(this->engine, this->topology_fused, bo_fused); @@ -40,6 +41,7 @@ public: network_not_fused.set_input_data("input", input_prim); compare(network_not_fused, network_fused, p); + check_fusions_correctness(network_fused, expected_fused_primitives_ids); } layout get_input_layout(resample_test_params& p) { @@ -308,3 +310,82 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, resample_scale_concat, ::testing::ValuesIn resample_test_params{ CASE_RESAMPLE_U8_3, 3, 6 }, resample_test_params{ CASE_RESAMPLE_U8_4, 3, 6 }, })); + +class resample_scale_fusing_through : public ResamplePrimitiveFusingTest {}; +TEST_P(resample_scale_fusing_through, reshape) { + auto p = GetParam(); + auto reshape_shape = p.out_shape; + reshape_shape.feature[0] *= reshape_shape.spatial[0]; + reshape_shape.spatial[0] = 1; + + create_topologies( + input_layout("input", get_input_layout(p)), + data("scale_data", get_mem(layout{ p.default_type, p.default_format, tensor{ 1, 1, 1, 1 } })), + resample("resample_prim", "input", p.out_shape, p.in_shape.feature[0], p.type), + reshape("reshape", "resample_prim", reshape_shape), + eltwise("scale", "reshape", "scale_data", eltwise_mode::prod), + reorder("reorder_bfyx", "scale", p.default_format, data_types::f32) + ); + + tolerance = 1e-5f; + execute(p, {{"resample_prim", {"scale"}}}); +} + +INSTANTIATE_TEST_SUITE_P(fusings_gpu, resample_scale_fusing_through, ::testing::ValuesIn(std::vector{ + resample_test_params{ CASE_RESAMPLE_FP32_1, 2, 3 }, + resample_test_params{ CASE_RESAMPLE_FP32_2, 2, 3 }, + resample_test_params{ CASE_RESAMPLE_FP32_3, 2, 3 }, + resample_test_params{ CASE_RESAMPLE_FP32_4, 2, 3 }, + resample_test_params{ CASE_RESAMPLE_FP32_5, 2, 3 }, + resample_test_params{ CASE_RESAMPLE_FP32_6, 2, 3 }, + resample_test_params{ CASE_RESAMPLE_FP32_7, 2, 3 }, + resample_test_params{ CASE_RESAMPLE_FP32_8, 2, 3 }, + + resample_test_params{ CASE_RESAMPLE_FP16_1, 2, 3 }, + resample_test_params{ CASE_RESAMPLE_FP16_2, 2, 3 }, + resample_test_params{ CASE_RESAMPLE_FP16_3, 2, 3 }, + resample_test_params{ CASE_RESAMPLE_FP16_4, 2, 3 }, + resample_test_params{ CASE_RESAMPLE_FP16_5, 2, 3 }, + resample_test_params{ CASE_RESAMPLE_FP16_6, 2, 3 }, + resample_test_params{ CASE_RESAMPLE_FP16_7, 2, 3 }, + resample_test_params{ CASE_RESAMPLE_FP16_8, 2, 3 }, + + resample_test_params{ CASE_RESAMPLE_I8_1, 2, 4 }, + resample_test_params{ CASE_RESAMPLE_I8_2, 2, 4 }, + resample_test_params{ CASE_RESAMPLE_I8_3, 2, 4 }, + resample_test_params{ CASE_RESAMPLE_I8_4, 2, 4 }, + + resample_test_params{ CASE_RESAMPLE_U8_1, 2, 4 }, + resample_test_params{ CASE_RESAMPLE_U8_2, 2, 4 }, + resample_test_params{ CASE_RESAMPLE_U8_3, 2, 4 }, + resample_test_params{ CASE_RESAMPLE_U8_4, 2, 4 }, +})); + +class resample_scale_fusing_through_not_allowed : public ResamplePrimitiveFusingTest {}; +TEST_P(resample_scale_fusing_through_not_allowed, reshape_two_users) { + auto p = GetParam(); + auto reshape_shape = p.out_shape; + reshape_shape.feature[0] *= reshape_shape.spatial[0]; + reshape_shape.spatial[0] = 1; + + create_topologies( + input_layout("input", get_input_layout(p)), + data("scale_data", get_mem(layout{ p.default_type, p.default_format, tensor{ 1, 1, 1, 1 } })), + resample("resample_prim", "input", p.out_shape, p.in_shape.feature[0], p.type), + reshape("reshape", "resample_prim", reshape_shape), + eltwise("scale", "reshape", "scale_data", eltwise_mode::prod), + eltwise("sum", "reshape", "scale", eltwise_mode::sum), + reorder("reorder_bfyx", "sum", p.default_format, data_types::f32) + ); + + tolerance = 1e-5f; + execute(p); +} + +INSTANTIATE_TEST_SUITE_P(fusings_gpu, resample_scale_fusing_through_not_allowed, ::testing::ValuesIn(std::vector{ + resample_test_params{ CASE_RESAMPLE_FP32_1, 4, 4 }, + resample_test_params{ CASE_RESAMPLE_FP32_2, 4, 4 }, + + resample_test_params{ CASE_RESAMPLE_FP16_1, 4, 4 }, + resample_test_params{ CASE_RESAMPLE_FP16_2, 4, 4 }, +})); diff --git a/src/plugins/intel_gpu/tests/fusions/softmax_fusion_test.cpp b/src/plugins/intel_gpu/tests/fusions/softmax_fusion_test.cpp new file mode 100644 index 00000000000..c71c2e4d155 --- /dev/null +++ b/src/plugins/intel_gpu/tests/fusions/softmax_fusion_test.cpp @@ -0,0 +1,171 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "test_utils.h" +#include "fusion_test_common.hpp" + +#include +#include +#include +#include +#include +#include + +#include + +using namespace cldnn; +using namespace ::tests; + +namespace { +struct softmax_test_params { + tensor in_shape; + data_types data_type; + format input_format; + softmax::dimension_t dimension; + data_types default_type; + format default_format; + size_t expected_fused_primitives; + size_t expected_not_fused_primitives; +}; + +class SoftmaxPrimitiveFusingTest : public ::BaseFusingTest { +public: + + void execute(softmax_test_params& p, std::map> expected_fused_primitives_ids = {}) { + auto input_prim = get_mem(get_input_layout(p)); + network network_not_fused(this->engine, this->topology_non_fused, bo_not_fused); + network network_fused(this->engine, this->topology_fused, bo_fused); + network_fused.set_input_data("input", input_prim); + network_not_fused.set_input_data("input", input_prim); + + compare(network_not_fused, network_fused, p); + check_fusions_correctness(network_fused, expected_fused_primitives_ids); + } + + layout get_input_layout(softmax_test_params& p) { + return layout{ p.data_type, p.input_format, p.in_shape, padding{} }; + } + + tensor get_reshape_shape(softmax_test_params& p) { + auto output_shape = p.in_shape; + output_shape.feature[0] *= output_shape.spatial[0]; + output_shape.spatial[0] = 1; + return output_shape; + } + + layout get_output_layout(softmax_test_params& p) { + return { data_types::f32, p.default_format, get_reshape_shape(p) }; + } +}; + +} // namespace + + +/* ----------------------------------------------------------------------------------------------------- */ +/* ---------------------------------------- SoftMax cases ---------------------------------------------- */ +/* ----------------------------------------------------------------------------------------------------- */ +#define CASE_SOFTMAX_FP32_1 {1, 15, 4, 5}, data_types::f32, format::bfyx, softmax::dimension_t::normalize_f, data_types::f32, format::bfyx +#define CASE_SOFTMAX_FP32_2 {1, 15, 4, 5}, data_types::f32, format::bfyx, softmax::dimension_t::normalize_x, data_types::f32, format::bfyx +#define CASE_SOFTMAX_FP32_3 {1, 15, 4, 5}, data_types::f32, format::bfyx, softmax::dimension_t::normalize_fyx, data_types::f32, format::bfyx + +#define CASE_SOFTMAX_FP16_1 {1, 15, 4, 5}, data_types::f16, format::bfyx, softmax::dimension_t::normalize_f, data_types::f16, format::bfyx +#define CASE_SOFTMAX_FP16_2 {1, 15, 4, 5}, data_types::f16, format::bfyx, softmax::dimension_t::normalize_x, data_types::f16, format::bfyx +#define CASE_SOFTMAX_FP16_3 {1, 15, 4, 5}, data_types::f16, format::bfyx, softmax::dimension_t::normalize_fyx, data_types::f16, format::bfyx + +class softmax_quantize : public SoftmaxPrimitiveFusingTest {}; +TEST_P(softmax_quantize, basic) { + auto p = GetParam(); + create_topologies(input_layout("input", get_input_layout(p)), + data("in_lo", get_mem(get_single_element_layout(p), min_random, 0)), + data("in_hi", get_mem(get_single_element_layout(p), 1, max_random)), + data("out_lo", get_mem(get_single_element_layout(p), -127)), + data("out_hi", get_mem(get_single_element_layout(p), 127)), + softmax("softmax", "input", p.dimension), + quantize("quantize", "softmax", "in_lo", "in_hi", "out_lo", "out_hi", 255, data_types::i8), + reorder("reorder_bfyx", "quantize", get_output_layout(p)) + ); + + tolerance = 1.f; + execute(p); +} + +INSTANTIATE_TEST_SUITE_P(fusings_gpu, softmax_quantize, + ::testing::ValuesIn(std::vector{ + softmax_test_params{ CASE_SOFTMAX_FP32_1, 2, 3 }, + softmax_test_params{ CASE_SOFTMAX_FP32_2, 3, 3 }, + softmax_test_params{ CASE_SOFTMAX_FP32_3, 3, 3 }, + + softmax_test_params{ CASE_SOFTMAX_FP16_1, 2, 3 }, + softmax_test_params{ CASE_SOFTMAX_FP16_2, 3, 3 }, + softmax_test_params{ CASE_SOFTMAX_FP16_3, 3, 3 }, +})); + +class softmax_quantize_fusing_through : public SoftmaxPrimitiveFusingTest {}; +TEST_P(softmax_quantize_fusing_through, reshape) { + auto p = GetParam(); + create_topologies(input_layout("input", get_input_layout(p)), + data("in_lo", get_mem(get_single_element_layout(p), min_random, 0)), + data("in_hi", get_mem(get_single_element_layout(p), 1, max_random)), + data("out_lo", get_mem(get_single_element_layout(p), -127)), + data("out_hi", get_mem(get_single_element_layout(p), 127)), + softmax("softmax", "input", p.dimension), + reshape("reshape", "softmax", get_reshape_shape(p)), + quantize("quantize", "reshape", "in_lo", "in_hi", "out_lo", "out_hi", 255, data_types::i8), + reorder("reorder_bfyx", "quantize", get_output_layout(p)) + ); + + tolerance = 1.f; + execute(p, {{"softmax", {"quantize"}}}); +} + +TEST_P(softmax_quantize_fusing_through, reorder) { + auto p = GetParam(); + auto reorder_layout = layout{ p.data_type, p.input_format, get_reshape_shape(p), padding{} }; + create_topologies(input_layout("input", get_input_layout(p)), + data("in_lo", get_mem(get_single_element_layout(p), min_random, 0)), + data("in_hi", get_mem(get_single_element_layout(p), 1, max_random)), + data("out_lo", get_mem(get_single_element_layout(p), -127)), + data("out_hi", get_mem(get_single_element_layout(p), 127)), + softmax("softmax", "input", p.dimension), + reorder("reorder", "softmax", reorder_layout), + quantize("quantize", "reorder", "in_lo", "in_hi", "out_lo", "out_hi", 255, data_types::i8), + reorder("reorder_bfyx", "quantize", get_output_layout(p)) + ); + + tolerance = 1.f; + execute(p, {{"softmax", {"quantize"}}}); +} + +TEST_P(softmax_quantize_fusing_through, chain) { + auto p = GetParam(); + auto reorder_layout = layout{ p.data_type, p.input_format, get_reshape_shape(p), padding{} }; + create_topologies(input_layout("input", get_input_layout(p)), + data("in_lo", get_mem(get_single_element_layout(p), min_random, 0)), + data("in_hi", get_mem(get_single_element_layout(p), 1, max_random)), + data("out_lo", get_mem(get_single_element_layout(p), -127)), + data("out_hi", get_mem(get_single_element_layout(p), 127)), + softmax("softmax", "input", p.dimension), + reshape("reshape_first", "softmax", get_reshape_shape(p)), + reorder("reorder", "reshape_first", reorder_layout), + reshape("reshape_second", "reorder", get_reshape_shape(p)), + quantize("quantize", "reshape_second", "in_lo", "in_hi", "out_lo", "out_hi", 255, data_types::i8), + reorder("reorder_bfyx", "quantize", get_output_layout(p)) + ); + + tolerance = 1.f; + execute(p, {{"softmax", {"quantize"}}}); +} + +INSTANTIATE_TEST_SUITE_P(fusings_gpu, softmax_quantize_fusing_through, + ::testing::ValuesIn(std::vector{ + softmax_test_params{ CASE_SOFTMAX_FP32_1, 2, 3 }, + // Such fusions not allowed yet + // softmax_test_params{ CASE_SOFTMAX_FP32_2, 3, 3 }, + // softmax_test_params{ CASE_SOFTMAX_FP32_3, 3, 3 }, + + softmax_test_params{ CASE_SOFTMAX_FP16_1, 2, 3 }, + // Such fusions not allowed yet + // softmax_test_params{ CASE_SOFTMAX_FP16_2, 3, 3 }, + // softmax_test_params{ CASE_SOFTMAX_FP16_3, 3, 3 }, +})); From ee31b648d19727a07b76cce703280d2215f06c00 Mon Sep 17 00:00:00 2001 From: Alexey Lebedev Date: Wed, 30 Mar 2022 17:03:29 +0300 Subject: [PATCH 03/39] [docs] port from release branch (#11309) * save work * Add common snipp * update ie pipeline with python snippets * ov_common_snippet * Python snippets for graph construction * Fix docs * Add missed old api snippets * Fix names * Fix markers * Fix methods call --- .../common_inference_pipeline.md | 302 +++++++++++++++++- .../migration_ov_2_0/configure_devices.md | 74 +++++ .../migration_ov_2_0/graph_construction.md | 24 +- docs/snippets/ie_common.py | 90 ++++++ docs/snippets/ngraph.py | 44 +++ docs/snippets/ov_common.py | 109 +++++++ docs/snippets/ov_graph.py | 46 +++ docs/snippets/ov_properties_migration.cpp | 4 +- docs/snippets/ov_properties_migration.py | 38 +++ 9 files changed, 726 insertions(+), 5 deletions(-) create mode 100644 docs/snippets/ie_common.py create mode 100644 docs/snippets/ngraph.py create mode 100644 docs/snippets/ov_common.py create mode 100644 docs/snippets/ov_graph.py diff --git a/docs/OV_Runtime_UG/migration_ov_2_0/common_inference_pipeline.md b/docs/OV_Runtime_UG/migration_ov_2_0/common_inference_pipeline.md index 7f3aa58d28b..75ce3495944 100644 --- a/docs/OV_Runtime_UG/migration_ov_2_0/common_inference_pipeline.md +++ b/docs/OV_Runtime_UG/migration_ov_2_0/common_inference_pipeline.md @@ -17,11 +17,31 @@ The following code shows how to change the application code in each step to migr Inference Engine API: +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ie_common.cpp ie:create_core +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ie_common.py ie:create_core +@endsphinxtab + +@endsphinxtabset OpenVINO™ Runtime API 2.0: +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ov_common.cpp ov_api_2_0:create_core +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_common.py ov_api_2_0:create_core +@endsphinxtab + +@endsphinxtabset ### 1.1 (Optional) Load extensions @@ -29,21 +49,61 @@ To load model with custom operation, you need to add extensions for these operat Inference Engine API: +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ie_common.cpp ie:load_old_extension +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ie_common.py ie:load_old_extension +@endsphinxtab + +@endsphinxtabset OpenVINO™ Runtime API 2.0: +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ov_common.cpp ov_api_2_0:load_old_extension +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_common.py ov_api_2_0:load_old_extension +@endsphinxtab + +@endsphinxtabset ## 2. Read model from the disk Inference Engine API: +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ie_common.cpp ie:read_model +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ie_common.py ie:read_model +@endsphinxtab + +@endsphinxtabset OpenVINO™ Runtime API 2.0: +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ov_common.cpp ov_api_2_0:read_model +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_common.py ov_api_2_0:read_model +@endsphinxtab + +@endsphinxtabset Read model has the same structure as in the example from [Model Creation](./graph_construction.md) migration guide. @@ -58,11 +118,31 @@ See detailed guide [how to migrate preprocessing in OpenVINO Runtime API 2.0](./ Inference Engine API: +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ie_common.cpp ie:compile_model +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ie_common.py ie:compile_model +@endsphinxtab + +@endsphinxtabset OpenVINO™ Runtime API 2.0: +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ov_common.cpp ov_api_2_0:compile_model +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_common.py ov_api_2_0:compile_model +@endsphinxtab + +@endsphinxtabset If you need to configure OpenVINO Runtime devices with additional configuration parameters, please, refer to the migration [Configure devices](./configure_devices.md) guide. @@ -70,11 +150,31 @@ If you need to configure OpenVINO Runtime devices with additional configuration Inference Engine API: +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ie_common.cpp ie:create_infer_request +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ie_common.py ie:create_infer_request +@endsphinxtab + +@endsphinxtabset OpenVINO™ Runtime API 2.0: +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ov_common.cpp ov_api_2_0:create_infer_request +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_common.py ov_api_2_0:create_infer_request +@endsphinxtab + +@endsphinxtabset ## 5. Fill input tensors @@ -84,25 +184,65 @@ Inference Engine API fills inputs as `I32` precision (**not** aligned with the o @sphinxtab{IR v10} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ie_common.cpp ie:get_input_tensor +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ie_common.py ie:get_input_tensor +@endsphinxtab + +@endsphinxtabset @endsphinxtab @sphinxtab{IR v11} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ie_common.cpp ie:get_input_tensor +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ie_common.py ie:get_input_tensor +@endsphinxtab + +@endsphinxtabset @endsphinxtab @sphinxtab{ONNX} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ie_common.cpp ie:get_input_tensor +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ie_common.py ie:get_input_tensor +@endsphinxtab + +@endsphinxtabset @endsphinxtab @sphinxtab{Model created in code} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ie_common.cpp ie:get_input_tensor +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ie_common.py ie:get_input_tensor +@endsphinxtab + +@endsphinxtabset @endsphinxtab @@ -114,25 +254,65 @@ OpenVINO™ Runtime API 2.0 fills inputs as `I64` precision (aligned with the or @sphinxtab{IR v10} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ov_common.cpp ov_api_2_0:get_input_tensor_v10 +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_common.py ov_api_2_0:get_input_tensor_v10 +@endsphinxtab + +@endsphinxtabset @endsphinxtab @sphinxtab{IR v11} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ov_common.cpp ov_api_2_0:get_input_tensor_aligned +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_common.py ov_api_2_0:get_input_tensor_aligned +@endsphinxtab + +@endsphinxtabset @endsphinxtab @sphinxtab{ONNX} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ov_common.cpp ov_api_2_0:get_input_tensor_aligned +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_common.py ov_api_2_0:get_input_tensor_aligned +@endsphinxtab + +@endsphinxtabset @endsphinxtab @sphinxtab{Model created in code} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ov_common.cpp ov_api_2_0:get_input_tensor_aligned +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_common.py ov_api_2_0:get_input_tensor_aligned +@endsphinxtab + +@endsphinxtabset @endsphinxtab @@ -146,13 +326,33 @@ Inference Engine API: @sphinxtab{Sync} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ie_common.cpp ie:inference +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ie_common.py ie:inference +@endsphinxtab + +@endsphinxtabset @endsphinxtab @sphinxtab{Async} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ie_common.cpp ie:start_async_and_wait +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ie_common.py ie:start_async_and_wait +@endsphinxtab + +@endsphinxtabset @endsphinxtab @@ -164,13 +364,33 @@ OpenVINO™ Runtime API 2.0: @sphinxtab{Sync} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ov_common.cpp ov_api_2_0:inference +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_common.py ov_api_2_0:inference +@endsphinxtab + +@endsphinxtabset @endsphinxtab @sphinxtab{Async} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ov_common.cpp ov_api_2_0:start_async_and_wait +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_common.py ov_api_2_0:start_async_and_wait +@endsphinxtab + +@endsphinxtabset @endsphinxtab @@ -184,25 +404,65 @@ Inference Engine API processes outputs as `I32` precision (**not** aligned with @sphinxtab{IR v10} -@snippet docs/snippets/ov_common.cpp ov_api_2_0:inference +@sphinxtabset + +@sphinxtab{C++} +@snippet docs/snippets/ie_common.cpp ie:get_output_tensor +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ie_common.py ie:get_output_tensor +@endsphinxtab + +@endsphinxtabset @endsphinxtab @sphinxtab{IR v11} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ie_common.cpp ie:get_output_tensor +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ie_common.py ie:get_output_tensor +@endsphinxtab + +@endsphinxtabset @endsphinxtab @sphinxtab{ONNX} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ie_common.cpp ie:get_output_tensor +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ie_common.py ie:get_output_tensor +@endsphinxtab + +@endsphinxtabset @endsphinxtab @sphinxtab{Model created in code} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ie_common.cpp ie:get_output_tensor +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ie_common.py ie:get_output_tensor +@endsphinxtab + +@endsphinxtabset @endsphinxtab @@ -216,25 +476,65 @@ OpenVINO™ Runtime API 2.0 processes outputs: @sphinxtab{IR v10} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ov_common.cpp ov_api_2_0:get_output_tensor_v10 +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_common.py ov_api_2_0:get_output_tensor_v10 +@endsphinxtab + +@endsphinxtabset @endsphinxtab @sphinxtab{IR v11} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ov_common.cpp ov_api_2_0:get_output_tensor_aligned +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_common.py ov_api_2_0:get_output_tensor_aligned +@endsphinxtab + +@endsphinxtabset @endsphinxtab @sphinxtab{ONNX} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ov_common.cpp ov_api_2_0:get_output_tensor_aligned +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_common.py ov_api_2_0:get_output_tensor_aligned +@endsphinxtab + +@endsphinxtabset @endsphinxtab @sphinxtab{Model created in code} +@sphinxtabset + +@sphinxtab{C++} @snippet docs/snippets/ov_common.cpp ov_api_2_0:get_output_tensor_aligned +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_common.py ov_api_2_0:get_output_tensor_aligned +@endsphinxtab + +@endsphinxtabset @endsphinxtab diff --git a/docs/OV_Runtime_UG/migration_ov_2_0/configure_devices.md b/docs/OV_Runtime_UG/migration_ov_2_0/configure_devices.md index fb773c58549..324763ea30d 100644 --- a/docs/OV_Runtime_UG/migration_ov_2_0/configure_devices.md +++ b/docs/OV_Runtime_UG/migration_ov_2_0/configure_devices.md @@ -22,6 +22,10 @@ Inference Engine API: @sphinxtabset +@sphinxtab{C++} + +@sphinxtabset + @sphinxtab{Devices} @snippet docs/snippets/ov_properties_migration.cpp core_set_config @@ -42,6 +46,36 @@ Inference Engine API: @endsphinxtabset +@endsphinxtab + +@sphinxtab{Python} + +@sphinxtabset + +@sphinxtab{Devices} + +@snippet docs/snippets/ov_properties_migration.py core_set_config + +@endsphinxtab + +@sphinxtab{Model Loading} + +@snippet docs/snippets/ov_properties_migration.py core_load_network + +@endsphinxtab + +@sphinxtab{Execution} + +@snippet docs/snippets/ov_properties_migration.py executable_network_set_config + +@endsphinxtab + +@endsphinxtabset + +@endsphinxtab + +@endsphinxtabset + OpenVINO Runtime API 2.0: @sphinxtabset @@ -106,6 +140,10 @@ Inference Engine API: @sphinxtabset +@sphinxtab{C++} + +@sphinxtabset + @sphinxtab{Device Configuration} @snippet docs/snippets/ov_properties_migration.cpp core_get_config @@ -132,6 +170,42 @@ Inference Engine API: @endsphinxtabset +@endsphinxtab + +@sphinxtab{Python} + +@sphinxtabset + +@sphinxtab{Device Configuration} + +@snippet docs/snippets/ov_properties_migration.py core_get_config + +@endsphinxtab + +@sphinxtab{Device metrics} + +@snippet docs/snippets/ov_properties_migration.py core_get_metric + +@endsphinxtab + +@sphinxtab{Execution config} + +@snippet docs/snippets/ov_properties_migration.py executable_network_get_config + +@endsphinxtab + +@sphinxtab{Execution metrics} + +@snippet docs/snippets/ov_properties_migration.py executable_network_get_metric + +@endsphinxtab + +@endsphinxtabset + +@endsphinxtab + +@endsphinxtabset + OpenVINO Runtime API 2.0: @sphinxtabset diff --git a/docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md b/docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md index f11318b0acb..b29350ed4fb 100644 --- a/docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md +++ b/docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md @@ -5,11 +5,31 @@ Code snippets below show how application code should be changed for migration to ### nGraph API -@snippet snippets/ngraph.cpp ngraph:graph +@sphinxtabset + +@sphinxtab{C++} +@snippet docs/snippets/ngraph.cpp ngraph:graph +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ngraph.py ngraph:graph +@endsphinxtab + +@endsphinxtabset ### OpenVINO™ Runtime API 2.0: -@snippet snippets/ov_graph.cpp ov:graph +@sphinxtabset + +@sphinxtab{C++} +@snippet docs/snippets/ov_graph.cpp ov:graph +@endsphinxtab + +@sphinxtab{Python} +@snippet docs/snippets/ov_graph.py ov:graph +@endsphinxtab + +@endsphinxtabset **See also:** - [Hello Model Creation C++ Sample](../../../samples/cpp/model_creation_sample/README.md) diff --git a/docs/snippets/ie_common.py b/docs/snippets/ie_common.py new file mode 100644 index 00000000000..74c3831ffd9 --- /dev/null +++ b/docs/snippets/ie_common.py @@ -0,0 +1,90 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + + +#! [ie:create_core] +import numpy as np +import openvino.inference_engine as ie +core = ie.IECore() +#! [ie:create_core] + +#! [ie:read_model] +network = core.read_network("model.xml") +#! [ie:read_model] + +#! [ie:compile_model] +# Load network to the device and create infer requests +exec_network = core.load_network(network, "CPU", num_requests=4) +#! [ie:compile_model] + +#! [ie:create_infer_request] +# Done in the previous step +#! [ie:create_infer_request] + +#! [ie:get_input_tensor] +infer_request = exec_network.requests[0] +# Get input blobs mapped to input layers names +input_blobs = infer_request.input_blobs +data = input_blobs["data1"].buffer +# Original I64 precision was converted to I32 +assert data.dtype == np.int32 +# Fill the first blob ... +#! [ie:get_input_tensor] + +#! [ie:inference] +results = infer_request.infer() +#! [ie:inference] + +input_data = iter(list()) + +def process_results(results, frame_id): + pass + +#! [ie:start_async_and_wait] +# Start async inference on a single infer request +infer_request.async_infer() +# Wait for 1 milisecond +infer_request.wait(1) +# Wait for inference completion +infer_request.wait() + +# Demonstrates async pipeline using ExecutableNetwork + +results = [] + +# Callback to process inference results +def callback(output_blobs, _): + # Copy the data from output blobs to numpy array + results_copy = {out_name: out_blob.buffer[:] for out_name, out_blob in output_blobs.items()} + results.append(process_results(results_copy)) + +# Setting callback for each infer requests +for infer_request in exec_network.requests: + infer_request.set_completion_callback(callback, py_data=infer_request.output_blobs) + +# Async pipline is managed by ExecutableNetwork +total_frames = 100 +for _ in range(total_frames): + # Wait for at least one free request + exec_network.wait(num_request=1) + # Get idle id + idle_id = exec_network.get_idle_request_id() + # Start asynchronous inference on idle request + exec_network.start_async(request_id=idle_id, inputs=next(input_data)) +# Wait for all requests to complete +exec_network.wait() +#! [ie:start_async_and_wait] + +#! [ie:get_output_tensor] +# Get output blobs mapped to output layers names +output_blobs = infer_request.output_blobs +data = output_blobs["out1"].buffer +# Original I64 precision was converted to I32 +assert data.dtype == np.int32 +# Process output data +#! [ie:get_output_tensor] + +#! [ie:load_old_extension] +core.add_extension("path_to_extension_library.so", "CPU") +#! [ie:load_old_extension] diff --git a/docs/snippets/ngraph.py b/docs/snippets/ngraph.py new file mode 100644 index 00000000000..7758fa8c6f4 --- /dev/null +++ b/docs/snippets/ngraph.py @@ -0,0 +1,44 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + + +#! [ngraph:graph] +# _____________ _____________ +# | Parameter | | Parameter | +# | data1 | | data2 | +# |___________| |___________| +# | | +# | | +# \ / +# \ / +# \ / +# ____\____/____ +# | Concat | +# | concat | +# |____________| +# | +# | +# | +# _______|_______ +# | Result | +# | result | +# |_____________| + +import ngraph as ng +import numpy as np + + +data1 = ng.opset8.parameter([1, 3, 2, 2], np.int64) +data1.friendly_name = "data1" # operation name +data2 = ng.opset8.parameter([1, 2, 2, 2], np.int64) +data2.friendly_name = "data2" # operation name + +concat = ng.opset8.concat([data1, data2], 1) +concat.friendly_name = "concat" # operation name + +result = ng.opset8.result(concat) +result.friendly_name = "result" # operation name + +f = ng.Function(result, [data1, data2], "function_name") +#! [ngraph:graph] diff --git a/docs/snippets/ov_common.py b/docs/snippets/ov_common.py new file mode 100644 index 00000000000..c6ea0a8f97d --- /dev/null +++ b/docs/snippets/ov_common.py @@ -0,0 +1,109 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + + +import numpy as np + +#! [ov_api_2_0:create_core] +import openvino.runtime as ov +core = ov.Core() +#! [ov_api_2_0:create_core] + +#! [ov_api_2_0:read_model] +model = core.read_model("model.xml") +#! [ov_api_2_0:read_model] + +#! [ov_api_2_0:compile_model] +compiled_model = core.compile_model(model, "CPU") +#! [ov_api_2_0:compile_model] + +#! [ov_api_2_0:create_infer_request] +infer_request = compiled_model.create_infer_request() +#! [ov_api_2_0:create_infer_request] + +#! [ov_api_2_0:get_input_tensor_aligned] +# Get input tensor by index +input_tensor1 = infer_request.get_input_tensor(0) +# Element types, names and layouts are aligned with framework +assert input_tensor1.data.dtype == np.int64 +# Fill the first data ... + +# Get input tensor by tensor name +input_tensor2 = infer_request.get_tensor("data2_t") +assert input_tensor2.data.dtype == np.int64 +# Fill the second data ... +#! [ov_api_2_0:get_input_tensor_aligned] + +#! [ov_api_2_0:get_input_tensor_v10] +# Get input tensor by index +input_tensor1 = infer_request.get_input_tensor(0) +# IR v10 works with converted precisions (i64 -> i32) +assert input_tensor1.data.dtype == np.int32 +# Fill the first data ... + +# Get input tensor by tensor name +input_tensor2 = infer_request.get_tensor("data2_t") +# IR v10 works with converted precisions (i64 -> i32) +assert input_tensor2.data.dtype == np.int32 +# Fill the second data .. +#! [ov_api_2_0:get_input_tensor_v10] + +#! [ov_api_2_0:inference] +results = infer_request.infer() +#! [ov_api_2_0:inference] + +input_data = iter(list()) + +def process_results(results, frame_id): + pass + +#! [ov_api_2_0:start_async_and_wait] +# Start async inference on a single infer request +infer_request.start_async() +# Wait for 1 milisecond +infer_request.wait_for(1) +# Wait for inference completion +infer_request.wait() + +# Demonstrates async pipeline using AsyncInferQueue + +results = [] + +def callback(request, frame_id): + # Copy the data from output tensors to numpy array and process it + results_copy = {output: data[:] for output, data in request.results.items()} + results.append(process_results(results_copy, frame_id)) + +# Create AsyncInferQueue with 4 infer requests +infer_queue = ov.AsyncInferQueue(compiled_model, jobs=4) +# Set callback for each infer request in the queue +infer_queue.set_callback(callback) + +total_frames = 100 +for i in range(total_frames): + # Wait for at least one available infer request and start asynchronous inference + infer_queue.start_async(next(input_data), userdata=i) +# Wait for all requests to complete +infer_queue.wait_all() +#! [ov_api_2_0:start_async_and_wait] + +#! [ov_api_2_0:get_output_tensor_aligned] +# Model has only one output +output_tensor = infer_request.get_output_tensor() +# Element types, names and layouts are aligned with framework +assert output_tensor.data.dtype == np.int64 +# process output data ... +#! [ov_api_2_0:get_output_tensor_aligned] + +#! [ov_api_2_0:get_output_tensor_v10] +# Model has only one output +output_tensor = infer_request.get_output_tensor() +# IR v10 works with converted precisions (i64 -> i32) +assert output_tensor.data.dtype == np.int32 +# process output data ... +#! [ov_api_2_0:get_output_tensor_v10] + +#! [ov_api_2_0:load_old_extension] +core.add_extension("path_to_extension_library.so") +#! [ov_api_2_0:load_old_extension] diff --git a/docs/snippets/ov_graph.py b/docs/snippets/ov_graph.py new file mode 100644 index 00000000000..e3eb6e9db45 --- /dev/null +++ b/docs/snippets/ov_graph.py @@ -0,0 +1,46 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + + +#! [ov:graph] +# _____________ _____________ +# | Parameter | | Parameter | +# | data1 | | data2 | +# |___________| |___________| +# | | +# data1_t | | data2_t +# \ / +# \ / +# \ / +# ____\____/____ +# | Concat | +# | concat | +# |____________| +# | +# | concat_t +# | +# _______|_______ +# | Result | +# | result | +# |_____________| + +import openvino.runtime as ov + + +data1 = ov.opset8.parameter([1, 3, 2, 2], ov.Type.i64) +data1.friendly_name = "data1" # operation name +data1.output(0).name = "data1_t" # tensor name +data2 = ov.opset8.parameter([1, 2, 2, 2], ov.Type.i64) +data2.friendly_name = "data2" # operation name +data2.output(0).name = "data2_t" # tensor name + +concat = ov.opset8.concat([data1, data2], 1) +concat.friendly_name = "concat" # operation name +concat.output(0).name = "concat_t" # tensor name + +result = ov.opset8.result(concat) +result.friendly_name = "result" # operation name + +model = ov.Model(result, [data1, data2], "model_name") +#! [ov:graph] diff --git a/docs/snippets/ov_properties_migration.cpp b/docs/snippets/ov_properties_migration.cpp index a982da7b29d..7be66b4a1d1 100644 --- a/docs/snippets/ov_properties_migration.cpp +++ b/docs/snippets/ov_properties_migration.cpp @@ -54,12 +54,12 @@ return 0; int main_old() { InferenceEngine::Core core; //! [core_get_metric] -auto full_device_name = core.GetConfig("CPU", METRIC_KEY(FULL_DEVICE_NAME)).as(); +auto full_device_name = core.GetMetric("CPU", METRIC_KEY(FULL_DEVICE_NAME)).as(); //! [core_get_metric] //! [core_get_config] // a user has to parse std::string after -auto num_streams = core.GetMetric("CPU", CONFIG_KEY(CPU_THROUGHPUT_STREAMS)).as(); +auto num_streams = core.GetConfig("CPU", CONFIG_KEY(CPU_THROUGHPUT_STREAMS)).as(); //! [core_get_config] //! [core_set_config] diff --git a/docs/snippets/ov_properties_migration.py b/docs/snippets/ov_properties_migration.py index a1030ae5c22..7cf7cc788fd 100644 --- a/docs/snippets/ov_properties_migration.py +++ b/docs/snippets/ov_properties_migration.py @@ -42,3 +42,41 @@ perf_mode = compiled_model.get_property("PERFORMANCE_HINT") # ! [compiled_model_get_ro_property] nireq = compiled_model.get_property("OPTIMAL_NUMBER_OF_INFER_REQUESTS") # ! [compiled_model_get_ro_property] + + +from openvino.inference_engine import IECore + + +core = IECore() +#! [core_get_metric] +full_device_name = core.get_metric("CPU", "FULL_DEVICE_NAME") +#! [core_get_metric] + +#! [core_get_config] +num_streams = core.get_config("CPU", "CPU_THROUGHPUT_STREAMS") +#! [core_get_config] + +#! [core_set_config] +core.set_config({"PERF_COUNT": "YES"}, "CPU") +#! [core_set_config] + +net = core.read_network("sample.xml") + +#! [core_load_network] +exec_network = core.load_network(net, "MULTI", {"DEVICE_PRIORITIES": "CPU, GPU", + "PERFORMANCE_HINT": "THROUGHPUT", + "ENFORCE_BF16": "NO"}) +#! [core_load_network] + +#! [executable_network_set_config] +# turn CPU off for multi-device execution +exec_network.set_config({"DEVICE_PRIORITIES": "GPU"}) +#! [executable_network_set_config] + +#! [executable_network_get_metric] +nireq = exec_network.get_metric("OPTIMAL_NUMBER_OF_INFER_REQUESTS") +#! [executable_network_get_metric] + +#! [executable_network_get_config] +perf_hint = exec_network.get_config("PERFORMANCE_HINT") +#! [executable_network_get_config] From 8317493e652dc23dae57f74ad70fd221a6af4abb Mon Sep 17 00:00:00 2001 From: Gleb Kazantaev Date: Wed, 30 Mar 2022 17:44:03 +0300 Subject: [PATCH 04/39] Update Accuracy Check inside Func tests (#11314) --- .../fq_mul_fusion_test.cpp | 4 +- .../fq_reshape_fusion.cpp | 4 +- .../matmul_const_transposes_extraction.cpp | 4 +- .../common_optimizations/nop_elimination.cpp | 4 +- .../common_optimizations/pad_fusion.cpp | 23 +- .../preprocessing_fusion_tests.cpp | 30 +- .../shuffle_channels_fusion_test.cpp | 4 +- .../common_optimizations/softmax_fusion.cpp | 8 +- .../transpose_sinking_test.cpp | 2 + .../transpose_to_reshape_test.cpp | 4 +- .../weights_dequantize_to_fake_quantize.cpp | 1 + .../legacy/conv_fusion_test.cpp | 4 +- .../legacy/mul_add_conversion_test.cpp | 8 +- .../offline_transformations/pruning_test.cpp | 40 +- .../convert_reduce_to_pooling_test.cpp | 4 +- ...nvert_scatter_elements_to_scatter_test.cpp | 4 +- .../utils/compare_functions_test.cpp | 5 +- .../utils/compress_quantize_weights.cpp | 10 +- .../common_test_utils/graph_comparator.cpp | 458 ++++++++++-------- .../common_test_utils/graph_comparator.hpp | 117 +++-- .../common_test_utils/ngraph_test_utils.cpp | 241 --------- .../common_test_utils/ngraph_test_utils.hpp | 20 +- 22 files changed, 422 insertions(+), 577 deletions(-) diff --git a/src/tests/functional/inference_engine/transformations/common_optimizations/fq_mul_fusion_test.cpp b/src/tests/functional/inference_engine/transformations/common_optimizations/fq_mul_fusion_test.cpp index 5b5287c789d..02bb9718e83 100644 --- a/src/tests/functional/inference_engine/transformations/common_optimizations/fq_mul_fusion_test.cpp +++ b/src/tests/functional/inference_engine/transformations/common_optimizations/fq_mul_fusion_test.cpp @@ -92,7 +92,9 @@ TEST_P(FQMulFusion, ExpectFusion) { manager.run_passes(m_function); ASSERT_NO_THROW(check_rt_info(m_function)); - auto fc = FunctionsComparator::no_default().enable(FunctionsComparator::PRECISIONS); + auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::PRECISIONS) + .enable(FunctionsComparator::NODES); auto res = fc.compare(m_function, m_expected_function); ASSERT_TRUE(res.valid) << res.message; }; diff --git a/src/tests/functional/inference_engine/transformations/common_optimizations/fq_reshape_fusion.cpp b/src/tests/functional/inference_engine/transformations/common_optimizations/fq_reshape_fusion.cpp index 3768f9cffda..a383e8913ff 100644 --- a/src/tests/functional/inference_engine/transformations/common_optimizations/fq_reshape_fusion.cpp +++ b/src/tests/functional/inference_engine/transformations/common_optimizations/fq_reshape_fusion.cpp @@ -113,7 +113,9 @@ TEST_P(nGraphFQReshapeFusionTests, ReshapeMatMul) { manager.run_passes(f); ASSERT_NO_THROW(check_rt_info(f)); - auto fc = FunctionsComparator::no_default().enable(FunctionsComparator::PRECISIONS); + auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::PRECISIONS) + .enable(FunctionsComparator::NODES); auto res = fc.compare(f, ref_f); ASSERT_TRUE(res.valid) << res.message; } diff --git a/src/tests/functional/inference_engine/transformations/common_optimizations/matmul_const_transposes_extraction.cpp b/src/tests/functional/inference_engine/transformations/common_optimizations/matmul_const_transposes_extraction.cpp index 76c2ce90aa1..39ea894dac6 100644 --- a/src/tests/functional/inference_engine/transformations/common_optimizations/matmul_const_transposes_extraction.cpp +++ b/src/tests/functional/inference_engine/transformations/common_optimizations/matmul_const_transposes_extraction.cpp @@ -30,7 +30,7 @@ TEST_F(TransformationTestsF, MatMulConstTransposesExtractionConstantWeights) { function_ref = std::make_shared(matmul, ParameterVector{data}); } comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, MatMulConstTransposesExtractionFQOnWeights) { @@ -57,7 +57,7 @@ TEST_F(TransformationTestsF, MatMulConstTransposesExtractionFQOnWeights) { function_ref = std::make_shared(matmul, ParameterVector{data}); } comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, NegativeMatMulConstTransposesExtractionInvalidRank) { diff --git a/src/tests/functional/inference_engine/transformations/common_optimizations/nop_elimination.cpp b/src/tests/functional/inference_engine/transformations/common_optimizations/nop_elimination.cpp index b38ab845172..c5da73d6edd 100644 --- a/src/tests/functional/inference_engine/transformations/common_optimizations/nop_elimination.cpp +++ b/src/tests/functional/inference_engine/transformations/common_optimizations/nop_elimination.cpp @@ -998,7 +998,7 @@ TEST_P(EliminateEltwiseTests, eliminate_eltwise) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); if (type == element::f32) { - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } } @@ -1088,5 +1088,5 @@ TEST_F(TransformationTestsF, eliminate_eltwise_dequantization_subgraph) { manager.register_pass(); comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } diff --git a/src/tests/functional/inference_engine/transformations/common_optimizations/pad_fusion.cpp b/src/tests/functional/inference_engine/transformations/common_optimizations/pad_fusion.cpp index 14efa6833a9..c9ab6da6805 100644 --- a/src/tests/functional/inference_engine/transformations/common_optimizations/pad_fusion.cpp +++ b/src/tests/functional/inference_engine/transformations/common_optimizations/pad_fusion.cpp @@ -120,11 +120,11 @@ TEST_F(TransformationTestsF, PadFusionConvolutionBackpropData) { auto pads_begin = opset5::Constant::create(element::i32, Shape{4}, {0, 0, 1, 1}); auto pads_end = opset5::Constant::create(element::i32, Shape{4}, {0, 0, 2, 2}); auto pad = std::make_shared(data, pads_begin, pads_end, op::PadMode::CONSTANT); - set_tensor_name(pad, "pad"); + auto filters = std::make_shared(element::f32, Shape{3, 2, 5, 5}); auto conv = std::make_shared(pad, filters, Strides{1, 1}, CoordinateDiff{4, 4}, CoordinateDiff{3, 3}, Shape{1, 1}); - set_tensor_name(conv, "conv"); + function = std::make_shared(NodeVector{conv}, ParameterVector{data, filters}); manager.register_pass(); @@ -134,7 +134,7 @@ TEST_F(TransformationTestsF, PadFusionConvolutionBackpropData) { auto filters = std::make_shared(element::f32, Shape{3, 2, 5, 5}); auto conv = std::make_shared(data, filters, Strides{1, 1}, CoordinateDiff{3, 3}, CoordinateDiff{1, 1}, Shape{1, 1}); - set_tensor_name(conv, "conv"); + function_ref = std::make_shared(NodeVector{conv}, ParameterVector{data, filters}); } @@ -244,11 +244,11 @@ TEST_F(TransformationTestsF, PadFusionConvolutionBackpropDataNonConstPadValue) { std::shared_ptr pad_value = opset5::Constant::create(element::f16, Shape{}, {0}); pad_value = std::make_shared(pad_value, element::f32); auto pad = std::make_shared(data, pads_begin, pads_end, pad_value, op::PadMode::CONSTANT); - set_tensor_name(pad, "pad"); + auto filters = std::make_shared(element::f32, Shape{3, 2, 5, 5}); auto conv = std::make_shared(pad, filters, Strides{1, 1}, CoordinateDiff{4, 4}, CoordinateDiff{3, 3}, Shape{1, 1}); - set_tensor_name(conv, "conv"); + function = std::make_shared(NodeVector{conv}, ParameterVector{data, filters}); manager.register_pass(); @@ -258,7 +258,7 @@ TEST_F(TransformationTestsF, PadFusionConvolutionBackpropDataNonConstPadValue) { auto filters = std::make_shared(element::f32, Shape{3, 2, 5, 5}); auto conv = std::make_shared(data, filters, Strides{1, 1}, CoordinateDiff{3, 3}, CoordinateDiff{1, 1}, Shape{1, 1}); - set_tensor_name(conv, "conv"); + function_ref = std::make_shared(NodeVector{conv}, ParameterVector{data, filters}); } @@ -422,30 +422,29 @@ TEST_F(TransformationTestsF, NegativePadFusionConvolutionBackpropDataTooSmallPad Shape data_shape{1, 3, 14, 14}; { auto data = std::make_shared(element::f32, data_shape); - set_tensor_name(data, "data"); + auto pads_begin = opset5::Constant::create(element::i32, Shape{4}, {0, 0, 2, 2}); auto pads_end = opset5::Constant::create(element::i32, Shape{4}, {0, 0, 2, 2}); auto pad = std::make_shared(data, pads_begin, pads_end, op::PadMode::CONSTANT); - set_tensor_name(pad, "pad"); + auto filters = std::make_shared(element::f32, Shape{3, 2, 5, 5}); auto conv = std::make_shared(pad, filters, Strides{1, 1}, CoordinateDiff{1, 1}, CoordinateDiff{1, 1}, Shape{1, 1}); - set_tensor_name(conv, "conv"); + function = std::make_shared(NodeVector{conv}, ParameterVector{data, filters}); manager.register_pass(); } { auto data = std::make_shared(element::f32, data_shape); - set_tensor_name(data, "data"); + auto pads_begin = opset5::Constant::create(element::i32, Shape{4}, {0, 0, 2, 2}); auto pads_end = opset5::Constant::create(element::i32, Shape{4}, {0, 0, 2, 2}); auto pad = std::make_shared(data, pads_begin, pads_end, op::PadMode::CONSTANT); - set_tensor_name(pad, "pad"); + auto filters = std::make_shared(element::f32, Shape{3, 2, 5, 5}); auto conv = std::make_shared(pad, filters, Strides{1, 1}, CoordinateDiff{1, 1}, CoordinateDiff{1, 1}, Shape{1, 1}); - set_tensor_name(conv, "conv"); function_ref = std::make_shared(NodeVector{conv}, ParameterVector{data, filters}); } diff --git a/src/tests/functional/inference_engine/transformations/common_optimizations/preprocessing_fusion_tests.cpp b/src/tests/functional/inference_engine/transformations/common_optimizations/preprocessing_fusion_tests.cpp index 700af83e1cf..4215ca2885a 100644 --- a/src/tests/functional/inference_engine/transformations/common_optimizations/preprocessing_fusion_tests.cpp +++ b/src/tests/functional/inference_engine/transformations/common_optimizations/preprocessing_fusion_tests.cpp @@ -141,7 +141,7 @@ TEST_F(TransformationTestsF, RICFusionSimple) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, RICFusionHard) { @@ -198,7 +198,7 @@ TEST_F(TransformationTestsF, RICFusionHard) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, RICFusionDynamic) { @@ -248,7 +248,7 @@ TEST_F(TransformationTestsF, RICFusionEltwise1) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, RICFusionEltwise2) { @@ -273,7 +273,7 @@ TEST_F(TransformationTestsF, RICFusionEltwise2) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, RICFusionEltwise3) { @@ -298,7 +298,7 @@ TEST_F(TransformationTestsF, RICFusionEltwise3) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, RICFusionEltwise4) { @@ -324,7 +324,7 @@ TEST_F(TransformationTestsF, RICFusionEltwise4) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, RICFusionEltwise5) { @@ -350,7 +350,7 @@ TEST_F(TransformationTestsF, RICFusionEltwise5) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, RICFusionEltwiseNegative) { @@ -390,7 +390,7 @@ TEST_F(TransformationTestsF, RICFusionEltwiseTwoRIC) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, RICFusionEltwiseNegative3) { @@ -431,7 +431,7 @@ TEST_F(TransformationTestsF, RICFusionGroupConv) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, RICFusionGroupConvNegative) { @@ -473,7 +473,7 @@ TEST_F(TransformationTestsF, RICFusionTranspose) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, RICFusionFQOnTheWay) { @@ -499,7 +499,7 @@ TEST_F(TransformationTestsF, RICFusionFQOnTheWay) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, RICFusionFQOnTheWay2) { @@ -538,7 +538,7 @@ TEST_F(TransformationTestsF, RICFusionFQOnTheWay2) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, RICFusionFQOnTheWay3) { @@ -578,7 +578,7 @@ TEST_F(TransformationTestsF, RICFusionFQOnTheWay3) { comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, RICFusionShapeOf) { @@ -749,7 +749,7 @@ TEST_F(TransformationTestsF, FuseScaleValue) { } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, FuseScaleValues) { @@ -779,5 +779,5 @@ TEST_F(TransformationTestsF, FuseScaleValues) { } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } diff --git a/src/tests/functional/inference_engine/transformations/common_optimizations/shuffle_channels_fusion_test.cpp b/src/tests/functional/inference_engine/transformations/common_optimizations/shuffle_channels_fusion_test.cpp index 54201fd589c..4ea58216808 100644 --- a/src/tests/functional/inference_engine/transformations/common_optimizations/shuffle_channels_fusion_test.cpp +++ b/src/tests/functional/inference_engine/transformations/common_optimizations/shuffle_channels_fusion_test.cpp @@ -90,7 +90,9 @@ protected: }; TEST_P(ShuffleChannelsFusion, CompareFunctions) { - auto fc = FunctionsComparator::no_default().enable(FunctionsComparator::PRECISIONS); + auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::PRECISIONS) + .enable(FunctionsComparator::NODES); auto res = fc.compare(f, f_ref); ASSERT_TRUE(res.valid) << res.message; } diff --git a/src/tests/functional/inference_engine/transformations/common_optimizations/softmax_fusion.cpp b/src/tests/functional/inference_engine/transformations/common_optimizations/softmax_fusion.cpp index 9ce13d0cae2..5ab4ccdb778 100644 --- a/src/tests/functional/inference_engine/transformations/common_optimizations/softmax_fusion.cpp +++ b/src/tests/functional/inference_engine/transformations/common_optimizations/softmax_fusion.cpp @@ -55,7 +55,9 @@ TEST_P(SoftmaxFusionFixture, SoftmaxFusion) { f_ref = std::make_shared(NodeVector{softmax}, ParameterVector{data}); } - auto fc = FunctionsComparator::no_default().enable(FunctionsComparator::PRECISIONS); + auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::PRECISIONS) + .enable(FunctionsComparator::NODES); auto res = fc.compare(f, f_ref); ASSERT_TRUE(res.valid) << res.message; } @@ -113,7 +115,9 @@ TEST_P(NegativeSoftmaxFusionFixture, NegativeSoftmaxFusion) { f_ref = std::make_shared(NodeVector{div}, ParameterVector{data}); } - auto fc = FunctionsComparator::no_default().enable(FunctionsComparator::PRECISIONS); + auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::PRECISIONS) + .enable(FunctionsComparator::NODES); auto res = fc.compare(f, f_ref); ASSERT_TRUE(res.valid) << res.message; } diff --git a/src/tests/functional/inference_engine/transformations/common_optimizations/transpose_sinking_test.cpp b/src/tests/functional/inference_engine/transformations/common_optimizations/transpose_sinking_test.cpp index 948a53db3d6..515e4ec59a6 100644 --- a/src/tests/functional/inference_engine/transformations/common_optimizations/transpose_sinking_test.cpp +++ b/src/tests/functional/inference_engine/transformations/common_optimizations/transpose_sinking_test.cpp @@ -94,6 +94,7 @@ TEST_P(TransposeSinkingFQ, TransposeFQReduce) { ASSERT_NO_THROW(check_rt_info(f)); auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::NODES) .enable(FunctionsComparator::PRECISIONS) .enable(FunctionsComparator::CONST_VALUES); auto res = fc.compare(f, f_ref); @@ -180,6 +181,7 @@ TEST_P(TransposeSinking, TransposeReduction) { ASSERT_NO_THROW(check_rt_info(f)); auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::NODES) .enable(FunctionsComparator::PRECISIONS) .enable(FunctionsComparator::CONST_VALUES); diff --git a/src/tests/functional/inference_engine/transformations/common_optimizations/transpose_to_reshape_test.cpp b/src/tests/functional/inference_engine/transformations/common_optimizations/transpose_to_reshape_test.cpp index 647ea874894..292ba8e88c3 100644 --- a/src/tests/functional/inference_engine/transformations/common_optimizations/transpose_to_reshape_test.cpp +++ b/src/tests/functional/inference_engine/transformations/common_optimizations/transpose_to_reshape_test.cpp @@ -109,7 +109,9 @@ TEST_P(TransposeToReshapeTests, CompareFunctions) { f->validate_nodes_and_infer_types(); ASSERT_NO_THROW(check_rt_info(f)); - auto fc = FunctionsComparator::no_default().enable(FunctionsComparator::PRECISIONS); + auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::NODES) + .enable(FunctionsComparator::PRECISIONS); auto res = fc.compare(f, f_ref); ASSERT_TRUE(res.valid) << res.message; } diff --git a/src/tests/functional/inference_engine/transformations/common_optimizations/weights_dequantize_to_fake_quantize.cpp b/src/tests/functional/inference_engine/transformations/common_optimizations/weights_dequantize_to_fake_quantize.cpp index 68f3883befc..a71c25dd933 100644 --- a/src/tests/functional/inference_engine/transformations/common_optimizations/weights_dequantize_to_fake_quantize.cpp +++ b/src/tests/functional/inference_engine/transformations/common_optimizations/weights_dequantize_to_fake_quantize.cpp @@ -84,6 +84,7 @@ TEST_P(TranslateNewWeightFormatToOldOne, ReshapeMatMul) { ASSERT_NO_THROW(check_rt_info(f)); auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::NODES) .enable(FunctionsComparator::PRECISIONS) .enable(FunctionsComparator::CONST_VALUES); auto res = fc.compare(f, f_ref); diff --git a/src/tests/functional/inference_engine/transformations/legacy/conv_fusion_test.cpp b/src/tests/functional/inference_engine/transformations/legacy/conv_fusion_test.cpp index 80f3488619e..a7858073da2 100644 --- a/src/tests/functional/inference_engine/transformations/legacy/conv_fusion_test.cpp +++ b/src/tests/functional/inference_engine/transformations/legacy/conv_fusion_test.cpp @@ -127,7 +127,9 @@ TEST_P(ConvFusionTests, CompareFunctions) { manager.run_passes(f); ASSERT_NO_THROW(check_rt_info(f)); - auto fc = FunctionsComparator::no_default().enable(FunctionsComparator::PRECISIONS); + auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::NODES) + .enable(FunctionsComparator::PRECISIONS); auto res = fc.compare(f, f_ref); ASSERT_TRUE(res.valid) << res.message; } diff --git a/src/tests/functional/inference_engine/transformations/legacy/mul_add_conversion_test.cpp b/src/tests/functional/inference_engine/transformations/legacy/mul_add_conversion_test.cpp index 1cd881caab3..1a511c7b4de 100644 --- a/src/tests/functional/inference_engine/transformations/legacy/mul_add_conversion_test.cpp +++ b/src/tests/functional/inference_engine/transformations/legacy/mul_add_conversion_test.cpp @@ -157,7 +157,9 @@ TEST_P(MulAddConversionTests, CompareFunctions) { ASSERT_NO_THROW(manager.run_passes(f)); f->validate_nodes_and_infer_types(); - auto fc = FunctionsComparator::no_default().enable(FunctionsComparator::PRECISIONS); + auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::NODES) + .enable(FunctionsComparator::PRECISIONS); auto res = fc.compare(f, f_ref); ASSERT_TRUE(res.valid) << res.message; } @@ -178,7 +180,9 @@ TEST_P(MulOrAddConversionTests, CompareFunctions) { f->validate_nodes_and_infer_types(); - auto fc = FunctionsComparator::no_default().enable(FunctionsComparator::PRECISIONS); + auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::NODES) + .enable(FunctionsComparator::PRECISIONS); auto res = fc.compare(f, f_ref); ASSERT_TRUE(res.valid) << res.message; } diff --git a/src/tests/functional/inference_engine/transformations/offline_transformations/pruning_test.cpp b/src/tests/functional/inference_engine/transformations/offline_transformations/pruning_test.cpp index f1ebb57e03f..e120936f679 100644 --- a/src/tests/functional/inference_engine/transformations/offline_transformations/pruning_test.cpp +++ b/src/tests/functional/inference_engine/transformations/offline_transformations/pruning_test.cpp @@ -246,7 +246,7 @@ TEST_F(TransformationTestsF, PropagateMasksBasic) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -311,7 +311,7 @@ TEST_F(TransformationTestsF, PropagateMasksDynamicConvolution) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -491,7 +491,7 @@ TEST_F(TransformationTestsF, PropagateMaskPassThrough) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -643,7 +643,7 @@ TEST_F(TransformationTestsF, PropagateMasksHardDependencies) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -761,7 +761,7 @@ TEST_F(TransformationTestsF, PropagateMasksQuantizedGroupConvolution) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -899,7 +899,7 @@ TEST_F(TransformationTestsF, PropagateMasksQuantizedGroupConvolutionWithShapeOf) m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -1017,7 +1017,7 @@ TEST_F(TransformationTestsF, PropagateMasksFakeQuantizePerTensor) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -1202,7 +1202,7 @@ TEST_F(TransformationTestsF, PropagateMasksFakeQuantizePerChannel) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -1299,7 +1299,7 @@ TEST_F(TransformationTestsF, TestConcatMaskPropagation) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -1406,7 +1406,7 @@ TEST_F(TransformationTestsF, TestConcatMaskPropagationUp) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -1554,7 +1554,7 @@ TEST_F(TransformationTestsF, PruneConvIsClosingAndInGroup) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -1814,7 +1814,7 @@ TEST_F(TransformationTestsF, PruneReduceLayerDown) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -1975,7 +1975,7 @@ TEST_F(TransformationTestsF, MaskPropagationReshapeUp) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -2050,7 +2050,7 @@ TEST_F(TransformationTestsF, MaskPropagationReshapeUpWithShapeOf) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -2149,7 +2149,7 @@ TEST_F(TransformationTestsF, MaskPropagationReshapeDown) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -2403,7 +2403,7 @@ TEST_F(TransformationTestsF, PruneSEBlock) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -2495,7 +2495,7 @@ TEST_F(TransformationTestsF, PropagateMasksLinear) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -2679,7 +2679,7 @@ TEST_F(TransformationTestsF, PruneMasksMatMulColsStopRowsUp) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -2760,7 +2760,7 @@ TEST_F(TransformationTestsF, PruneMasksMatMulRowsStopColsUp) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } @@ -2855,5 +2855,5 @@ TEST_F(TransformationTestsF, PropagateFlattenUp) { m.run_passes(function); } disable_rt_info_check(); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } diff --git a/src/tests/functional/inference_engine/transformations/op_conversions/convert_reduce_to_pooling_test.cpp b/src/tests/functional/inference_engine/transformations/op_conversions/convert_reduce_to_pooling_test.cpp index d27e7d4f74f..16a35fdfa81 100644 --- a/src/tests/functional/inference_engine/transformations/op_conversions/convert_reduce_to_pooling_test.cpp +++ b/src/tests/functional/inference_engine/transformations/op_conversions/convert_reduce_to_pooling_test.cpp @@ -126,7 +126,9 @@ TEST_P(ConvertReduceToPoolingTests, CompareFunctions) { m.run_passes(f); ASSERT_NO_THROW(check_rt_info(f)); - auto fc = FunctionsComparator::no_default().enable(FunctionsComparator::PRECISIONS); + auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::NODES) + .enable(FunctionsComparator::PRECISIONS); auto res = fc.compare(f, f_ref); ASSERT_TRUE(res.valid) << res.message; } diff --git a/src/tests/functional/inference_engine/transformations/op_conversions/convert_scatter_elements_to_scatter_test.cpp b/src/tests/functional/inference_engine/transformations/op_conversions/convert_scatter_elements_to_scatter_test.cpp index eb684415335..c626ff597b0 100644 --- a/src/tests/functional/inference_engine/transformations/op_conversions/convert_scatter_elements_to_scatter_test.cpp +++ b/src/tests/functional/inference_engine/transformations/op_conversions/convert_scatter_elements_to_scatter_test.cpp @@ -90,7 +90,9 @@ void test(std::shared_ptr f, std::shared_ptr manager.register_pass(); ASSERT_NO_THROW(manager.run_passes(f)); - auto fc = FunctionsComparator::no_default().enable(FunctionsComparator::PRECISIONS); + auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::NODES) + .enable(FunctionsComparator::PRECISIONS); auto res = fc.compare(f, f_ref); ASSERT_TRUE(res.valid) << res.message; } diff --git a/src/tests/functional/inference_engine/transformations/utils/compare_functions_test.cpp b/src/tests/functional/inference_engine/transformations/utils/compare_functions_test.cpp index e380f5045ac..8555ab1658b 100644 --- a/src/tests/functional/inference_engine/transformations/utils/compare_functions_test.cpp +++ b/src/tests/functional/inference_engine/transformations/utils/compare_functions_test.cpp @@ -650,7 +650,9 @@ TEST(TransformationTests, DifferentPrecisionVersusAttributes) { /// { // check precision only - const auto fc = FunctionsComparator::no_default().enable(FunctionsComparator::PRECISIONS); + const auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::NODES) + .enable(FunctionsComparator::PRECISIONS); const auto res = fc.compare(f1, f2); EXPECT_FALSE(res.valid); EXPECT_THAT(res.message, HasSubstr("Different element type detected")); @@ -660,6 +662,7 @@ TEST(TransformationTests, DifferentPrecisionVersusAttributes) { { // check precision and attributes const auto fc = FunctionsComparator::no_default() + .enable(FunctionsComparator::NODES) .enable(FunctionsComparator::PRECISIONS) .enable(FunctionsComparator::ATTRIBUTES); const auto res = fc.compare(f1, f2); diff --git a/src/tests/functional/inference_engine/transformations/utils/compress_quantize_weights.cpp b/src/tests/functional/inference_engine/transformations/utils/compress_quantize_weights.cpp index 3c5310a666f..4b869521c53 100644 --- a/src/tests/functional/inference_engine/transformations/utils/compress_quantize_weights.cpp +++ b/src/tests/functional/inference_engine/transformations/utils/compress_quantize_weights.cpp @@ -62,7 +62,7 @@ class CompressQuantizeWeightsTests function_ref = std::make_shared(mul, ParameterVector{}); } comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } }; @@ -108,7 +108,7 @@ TEST_F(TransformationTestsF, CompressQuantizeWeightsWithDequantizationSubgraph) function_ref = std::make_shared(NodeVector{mul}, ParameterVector{}); } comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, CompressQuantizeWeightsWithZeroPointOptimizer) { @@ -133,7 +133,7 @@ TEST_F(TransformationTestsF, CompressQuantizeWeightsWithZeroPointOptimizer) { function_ref = std::make_shared(NodeVector{mul}, ParameterVector{}); } comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, NegativeCompressQuantizeWeightsWithZeroPointOptimizer) { @@ -159,7 +159,7 @@ TEST_F(TransformationTestsF, NegativeCompressQuantizeWeightsWithZeroPointOptimiz function_ref = std::make_shared(NodeVector{mul}, ParameterVector{}); } comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } TEST_F(TransformationTestsF, NegativeCompressQuantizeWeightsNonConstantInput) { @@ -175,5 +175,5 @@ TEST_F(TransformationTestsF, NegativeCompressQuantizeWeightsNonConstantInput) { manager.register_pass(); comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); - enable_accuracy_check(); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } diff --git a/src/tests/ie_test_utils/common_test_utils/graph_comparator.cpp b/src/tests/ie_test_utils/common_test_utils/graph_comparator.cpp index 76a72195110..3469cbe37ee 100644 --- a/src/tests/ie_test_utils/common_test_utils/graph_comparator.cpp +++ b/src/tests/ie_test_utils/common_test_utils/graph_comparator.cpp @@ -2,7 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "graph_comparator.hpp" +#include "common_test_utils/graph_comparator.hpp" + +#include +#include "ie_common.h" #include #include @@ -20,6 +23,10 @@ #include #include +#include "common_test_utils/ov_tensor_utils.hpp" +#include "ngraph_functions/utils/ngraph_helpers.hpp" + + namespace { inline namespace tools { bool is_type_relaxed(const std::string& type) { @@ -31,12 +38,12 @@ bool compare_type_info(const ngraph::DiscreteTypeInfo& info1, const ngraph::Disc if (!is_type_relaxed(info1.name) && !is_type_relaxed(info2.name) && (info1.version != info2.version)) { return false; } - OPENVINO_SUPPRESS_DEPRECATED_END + OPENVINO_SUPPRESS_DEPRECATED_END const std::string info1Name = - is_type_relaxed(info1.name) && (info1.parent != nullptr) ? info1.parent->name : info1.name; + is_type_relaxed(info1.name) && (info1.parent != nullptr) ? info1.parent->name : info1.name; const std::string info2Name = - is_type_relaxed(info2.name) && (info2.parent != nullptr) ? info2.parent->name : info2.name; + is_type_relaxed(info2.name) && (info2.parent != nullptr) ? info2.parent->name : info2.name; return info1Name == info2Name; } @@ -121,9 +128,9 @@ public: explicit NodeAndInputDescription(const InputNode& input, const Parameter* parameter, const InputDescripton* description) - : m_input(input), - m_parameter(not_null(parameter)), - m_description(not_null(description)) {} + : m_input(input), + m_parameter(not_null(parameter)), + m_description(not_null(description)) {} static bool equal_descriptions(const InputDescripton* lhs, const InputDescripton* rhs) { if (!lhs || !rhs || lhs->get_type_info() != rhs->get_type_info()) { @@ -172,7 +179,7 @@ public: } for (size_t i = 0; i != param_shape.size(); ++i) { const auto expected_axis_size = - i == slice_description->m_axis ? slice_description->m_part_size * num_iterations : param_shape[i]; + i == slice_description->m_axis ? slice_description->m_part_size * num_iterations : param_shape[i]; if (input_shape[i] != expected_axis_size) { return false; } @@ -215,9 +222,9 @@ public: explicit NodeAndOutputDescription(const OutputNode& output, const Result* result, const OutputDescription* description) - : m_output(output), - m_result(not_null(result)), - m_description(not_null(description)) {} + : m_output(output), + m_result(not_null(result)), + m_description(not_null(description)) {} static bool equal_descriptions(const OutputDescription* lhs, const OutputDescription* rhs) { if (!lhs || !rhs || lhs->get_type_info() != rhs->get_type_info()) { @@ -302,8 +309,8 @@ public: using Id = uint64_t; explicit BackEdge(const Parameter* parameter, const Result* result) - : m_parameter(not_null(parameter)), - m_result(not_null(result)) {} + : m_parameter(not_null(parameter)), + m_result(not_null(result)) {} bool result_and_parameter_match() const { return equal_type_and_partial_shape(m_result->output(0), *m_parameter); @@ -539,103 +546,113 @@ Comparator::Result Comparator::compare(const std::shared_ptr& * + Check node attributes by Visitor API */ - auto f_results = f->get_results(); - auto f_ref_results = f_ref->get_results(); + if (should_compare(CmpValues::NODES)) { + auto f_results = f->get_results(); + auto f_ref_results = f_ref->get_results(); - auto cmp = less_by_name; - // In case if Result source output has more than one name so the Result may have any of this names as a friendly - // name An in case of multiple names we sort Result operation using their parent node names - if (std::any_of(f_results.begin(), - f_results.end(), - [](const std::shared_ptr& node) { - const auto& t = node->input_value(0).get_tensor_ptr(); - return t->get_names().size() > 1; - }) || - std::any_of(f_ref_results.begin(), f_ref_results.end(), [](const std::shared_ptr& node) { - const auto& t = node->input_value(0).get_tensor_ptr(); - return t->get_names().size() > 1; - })) { - cmp = less_by_parent_name; - } + auto cmp = less_by_name; + // In case if Result source output has more than one name so the Result may have any of this names as a friendly + // name An in case of multiple names we sort Result operation using their parent node names + if (std::any_of(f_results.begin(), + f_results.end(), + [](const std::shared_ptr &node) { + const auto &t = node->input_value(0).get_tensor_ptr(); + return t->get_names().size() > 1; + }) || + std::any_of(f_ref_results.begin(), f_ref_results.end(), [](const std::shared_ptr &node) { + const auto &t = node->input_value(0).get_tensor_ptr(); + return t->get_names().size() > 1; + })) { + cmp = less_by_parent_name; + } - std::sort(f_results.begin(), f_results.end(), cmp); - std::sort(f_ref_results.begin(), f_ref_results.end(), cmp); + std::sort(f_results.begin(), f_results.end(), cmp); + std::sort(f_ref_results.begin(), f_ref_results.end(), cmp); - if (f_results.size() != f_ref_results.size()) { - return Result::error("Number of results is different: " + to_str(f_results.size()) + " and " + - to_str(f_ref_results.size())); - } + if (f_results.size() != f_ref_results.size()) { + return Result::error("Number of results is different: " + to_str(f_results.size()) + " and " + + to_str(f_ref_results.size())); + } - const auto& f_sinks = f->get_sinks(); - const auto& f_ref_sinks = f_ref->get_sinks(); - if (f_sinks.size() != f_ref_sinks.size()) { - return Result::error("Number of sinks is different: " + to_str(f_sinks.size()) + " and " + - to_str(f_ref_sinks.size())); - } + const auto &f_sinks = f->get_sinks(); + const auto &f_ref_sinks = f_ref->get_sinks(); + if (f_sinks.size() != f_ref_sinks.size()) { + return Result::error("Number of sinks is different: " + to_str(f_sinks.size()) + " and " + + to_str(f_ref_sinks.size())); + } - // Compare sinks - if (f_sinks.size() == 1) { - q.push({f_sinks[0].get(), f_ref_sinks[0].get()}); - used.insert(f_sinks[0].get()); - } else { - // Cast to Assign and find those that have same variable_id suffix - for (const auto& sink1 : f_sinks) { - auto assign1 = std::dynamic_pointer_cast(sink1); - if (!assign1) { - return Result::error("Sink '" + name(sink1) + - "' is not a variable - graph comparison is not supported"); - } - auto name1 = assign1->get_variable_id(); - std::shared_ptr found_sink2; - for (const auto& sink2 : f_ref_sinks) { - auto assign2 = std::dynamic_pointer_cast(sink2); - if (!assign2) { - return Result::error("Sink '" + name(sink2) + + // Compare sinks + if (f_sinks.size() == 1) { + q.push({f_sinks[0].get(), f_ref_sinks[0].get()}); + used.insert(f_sinks[0].get()); + } else { + // Cast to Assign and find those that have same variable_id suffix + for (const auto &sink1 : f_sinks) { + auto assign1 = std::dynamic_pointer_cast(sink1); + if (!assign1) { + return Result::error("Sink '" + name(sink1) + "' is not a variable - graph comparison is not supported"); } - auto name2 = assign2->get_variable_id(); - if (name2.find(name1) != std::string::npos || name1.find(name2) != std::string::npos) { - found_sink2 = sink2; - break; + auto name1 = assign1->get_variable_id(); + std::shared_ptr found_sink2; + for (const auto &sink2 : f_ref_sinks) { + auto assign2 = std::dynamic_pointer_cast(sink2); + if (!assign2) { + return Result::error("Sink '" + name(sink2) + + "' is not a variable - graph comparison is not supported"); + } + auto name2 = assign2->get_variable_id(); + if (name2.find(name1) != std::string::npos || name1.find(name2) != std::string::npos) { + found_sink2 = sink2; + break; + } + } + if (!found_sink2) { + return Result::error("No suitable sink is found for: " + name(sink1) + ", var=" + name1); + } + q.push({sink1.get(), found_sink2.get()}); + used.insert(sink1.get()); + } + } + + for (size_t i = 0; i < f_results.size(); ++i) { + if (should_compare(CmpValues::NAMES)) { + if (name(f_results[i]->get_input_node_shared_ptr(0)) != + name(f_ref_results[i]->get_input_node_shared_ptr(0))) { + return Result::error( + "Different output node names: " + name(f_results[i]->get_input_node_shared_ptr(0)) + + " and " + + name(f_ref_results[i]->get_input_node_shared_ptr(0))); } } - if (!found_sink2) { - return Result::error("No suitable sink is found for: " + name(sink1) + ", var=" + name1); + q.push({f_results[i].get(), f_ref_results[i].get()}); + used.insert(f_results[i].get()); + } + + std::stringstream errors; + + while (!q.empty()) { + ngraph::Node *const node1 = q.front().first; + ngraph::Node *const node2 = q.front().second; + q.pop(); + + const auto result = compare(node1, node2, errors); + if (!result.valid) { + return result; } - q.push({sink1.get(), found_sink2.get()}); - used.insert(sink1.get()); + + add_nodes_inputs_to_queue(node1, node2); } + const auto msg = errors.str(); + return msg.empty() ? Result::ok() : Result::error(msg); + + } else if (should_compare(CmpValues::ACCURACY)) { + auto status = accuracy_check(f_ref, f); + return status.status ? Result::ok() : Result::error(status.message); + + } else { + return Result::error("CmpValues are invalid"); } - - for (size_t i = 0; i < f_results.size(); ++i) { - if (should_compare(CmpValues::NAMES)) { - if (name(f_results[i]->get_input_node_shared_ptr(0)) != - name(f_ref_results[i]->get_input_node_shared_ptr(0))) { - return Result::error( - "Different output node names: " + name(f_results[i]->get_input_node_shared_ptr(0)) + " and " + - name(f_ref_results[i]->get_input_node_shared_ptr(0))); - } - } - q.push({f_results[i].get(), f_ref_results[i].get()}); - used.insert(f_results[i].get()); - } - - std::stringstream errors; - - while (!q.empty()) { - ngraph::Node* const node1 = q.front().first; - ngraph::Node* const node2 = q.front().second; - q.pop(); - - const auto result = compare(node1, node2, errors); - if (!result.valid) { - return result; - } - - add_nodes_inputs_to_queue(node1, node2); - } - const auto msg = errors.str(); - return msg.empty() ? Result::ok() : Result::error(msg); } Comparator::Result Comparator::compare(ngraph::Node* node1, ngraph::Node* node2, std::ostream& err_log) { @@ -759,7 +776,7 @@ void Comparator::compare_outputs(ngraph::Node* node1, ngraph::Node* node2, std:: void Comparator::compare_nodes(ngraph::Node* node1, ngraph::Node* node2, std::ostream& err_log) { if (should_compare(CmpValues::RUNTIME_KEYS) && !compare_rt_keys(*node1, *node2, err_log)) { err_log << "Different runtime info detected\n" + name(node1) + " and " + name(node2) + - " not equal runtime info.\n"; + " not equal runtime info.\n"; } if (should_compare(CmpValues::ATTRIBUTES)) { @@ -805,123 +822,115 @@ void check_rt_info(const std::shared_ptr& f) { } } -void set_tensor_name(ngraph::Output output, const std::string& name) { - output.get_tensor_ptr()->set_names({name}); -} - -void set_tensor_names(ngraph::Output output, const std::unordered_set& names) { - output.get_tensor_ptr()->set_names(names); -} - namespace attributes { namespace detail { -void ReadAndStoreAttributes::on_adapter(const std::string& name, ngraph::ValueAccessor& adapter) { - if (auto inputs = ngraph::as_type>(&adapter)) { - insert(name, inputs->get()); - } else if (auto outputs = ngraph::as_type>(&adapter)) { - insert(name, outputs->get()); - } else if (ngraph::is_type>(&adapter)) { - // drop comparison, no more info than port indexes which will be check in - // subgraph::compare_io - } else if (auto a = ngraph::as_type>>( - &adapter)) { - const auto beg = static_cast(a->get()->get_ptr()); - const auto end = beg + a->get()->size(); - insert(name, storage::MemoryChunk{storage::MemoryChunk::Data(beg, end)}); - } else if (auto framework_node_attr = - ngraph::as_type>(&adapter)) { - insert(name, framework_node_attr->get()); - } else if (auto variable_ptr = - ngraph::as_type>>(&adapter)) { - insert(name, variable_ptr->get()); - } else if (auto shape_ptr = ngraph::as_type>(&adapter)) { - insert(name, shape_ptr->get()); - } else if (auto dim_ptr = ngraph::as_type>(&adapter)) { - insert(name, dim_ptr->get()); - } else { - m_read_result += "store attr [ ERR ]: " + name + " [drop `void` comparison which is '" + - adapter.get_type_info().name + "']"; + void ReadAndStoreAttributes::on_adapter(const std::string& name, ngraph::ValueAccessor& adapter) { + if (auto inputs = ngraph::as_type>(&adapter)) { + insert(name, inputs->get()); + } else if (auto outputs = ngraph::as_type>(&adapter)) { + insert(name, outputs->get()); + } else if (ngraph::is_type>(&adapter)) { + // drop comparison, no more info than port indexes which will be check in + // subgraph::compare_io + } else if (auto a = ngraph::as_type>>( + &adapter)) { + const auto beg = static_cast(a->get()->get_ptr()); + const auto end = beg + a->get()->size(); + insert(name, storage::MemoryChunk{storage::MemoryChunk::Data(beg, end)}); + } else if (auto framework_node_attr = + ngraph::as_type>(&adapter)) { + insert(name, framework_node_attr->get()); + } else if (auto variable_ptr = + ngraph::as_type>>(&adapter)) { + insert(name, variable_ptr->get()); + } else if (auto shape_ptr = ngraph::as_type>(&adapter)) { + insert(name, shape_ptr->get()); + } else if (auto dim_ptr = ngraph::as_type>(&adapter)) { + insert(name, dim_ptr->get()); + } else { + m_read_result += "store attr [ ERR ]: " + name + " [drop `void` comparison which is '" + + adapter.get_type_info().name + "']"; + } } -} -template -void ReadAndCompareAttributes::verify(const std::string& name, const AttrValue& attr_value) { - if (should_return()) { - return; - } - m_visited_attributes.insert(name); - const auto ref_value = m_attr_ref.get(name); - if (!ref_value) { - m_cmp_result += "missing attribute name: '" + name + "'"; - return; + template + void ReadAndCompareAttributes::verify(const std::string& name, const AttrValue& attr_value) { + if (should_return()) { + return; + } + m_visited_attributes.insert(name); + const auto ref_value = m_attr_ref.get(name); + if (!ref_value) { + m_cmp_result += "missing attribute name: '" + name + "'"; + return; + } + + if (!equal::Equal::equal_value(*ref_value, attr_value)) { + m_cmp_result += "mismatch in value: '" + name + "' : " + str::Get::value(*ref_value) + " vs " + + str::Get::value(attr_value); + } } - if (!equal::Equal::equal_value(*ref_value, attr_value)) { - m_cmp_result += "mismatch in value: '" + name + "' : " + str::Get::value(*ref_value) + " vs " + - str::Get::value(attr_value); - } -} + void ReadAndCompareAttributes::verify_mem_buf(const std::string& name, + const std::shared_ptr& buffer) { + if (should_return()) { + return; + } + m_visited_attributes.insert(name); + const auto ref_value = m_attr_ref.get(name); + if (!ref_value) { + m_cmp_result += "missing attribute name: '" + name + "'"; + return; + } -void ReadAndCompareAttributes::verify_mem_buf(const std::string& name, - const std::shared_ptr& buffer) { - if (should_return()) { - return; - } - m_visited_attributes.insert(name); - const auto ref_value = m_attr_ref.get(name); - if (!ref_value) { - m_cmp_result += "missing attribute name: '" + name + "'"; - return; + if (buffer->size() != ref_value->size() || + std::memcmp(ref_value->data(), buffer->get_ptr(), ref_value->size()) != 0) { + m_cmp_result += "mismatch in value: '" + name + "' : look in to the mem buffer"; + return; + } } - if (buffer->size() != ref_value->size() || - std::memcmp(ref_value->data(), buffer->get_ptr(), ref_value->size()) != 0) { - m_cmp_result += "mismatch in value: '" + name + "' : look in to the mem buffer"; - return; + void ReadAndCompareAttributes::verify_function(const std::string& name, ModelAccessor& adapter) { + if (should_return()) { + return; + } + m_visited_attributes.insert(name); + const auto ref_value = m_attr_ref.get>(name); + if (!ref_value) { + m_cmp_result += "missing attribute name: '" + name + "'"; + return; + } + Comparator c(m_check_flags); + const auto result = c.compare(*ref_value, adapter.get()); + if (!result.valid) { + m_cmp_result += result.message; + } } -} -void ReadAndCompareAttributes::verify_function(const std::string& name, ModelAccessor& adapter) { - if (should_return()) { - return; + void ReadAndCompareAttributes::verify_others(const std::string& name, ngraph::ValueAccessor& adapter) { + if (auto inputs = ngraph::as_type>(&adapter)) { + verify(name, inputs->get()); + } else if (auto outputs = ngraph::as_type>(&adapter)) { + verify(name, outputs->get()); + } else if (ngraph::is_type>(&adapter)) { + // drop comparison, no more info than port indexes which will be check in + // subgraph::compare_io + } else if (auto a = ngraph::as_type>>( + &adapter)) { + verify_mem_buf(name, a->get()); + } else if (auto attrs = ngraph::as_type>(&adapter)) { + verify(name, attrs->get()); + } else if (auto variable_ptr = + ngraph::as_type>>(&adapter)) { + verify(name, variable_ptr->get()); + } else if (auto shape_ptr = ngraph::as_type>(&adapter)) { + verify(name, shape_ptr->get()); + } else if (auto dim_ptr = ngraph::as_type>(&adapter)) { + verify(name, dim_ptr->get()); + } else { + m_cmp_result += "compare attr [ ERR ]: " + name + " [drop `void` comparison which is '" + + adapter.get_type_info().name + "']"; + } } - m_visited_attributes.insert(name); - const auto ref_value = m_attr_ref.get>(name); - if (!ref_value) { - m_cmp_result += "missing attribute name: '" + name + "'"; - return; - } - Comparator c(m_check_flags); - const auto result = c.compare(*ref_value, adapter.get()); - if (!result.valid) { - m_cmp_result += result.message; - } -} - -void ReadAndCompareAttributes::verify_others(const std::string& name, ngraph::ValueAccessor& adapter) { - if (auto inputs = ngraph::as_type>(&adapter)) { - verify(name, inputs->get()); - } else if (auto outputs = ngraph::as_type>(&adapter)) { - verify(name, outputs->get()); - } else if (ngraph::is_type>(&adapter)) { - // drop comparison, no more info than port indexes which will be check in - // subgraph::compare_io - } else if (auto a = ngraph::as_type>>( - &adapter)) { - verify_mem_buf(name, a->get()); - } else if (auto attrs = ngraph::as_type>(&adapter)) { - verify(name, attrs->get()); - } else if (auto variable_ptr = - ngraph::as_type>>(&adapter)) { - verify(name, variable_ptr->get()); - } else if (auto shape_ptr = ngraph::as_type>(&adapter)) { - verify(name, shape_ptr->get()); - } else if (auto dim_ptr = ngraph::as_type>(&adapter)) { - verify(name, dim_ptr->get()); - } else { - m_cmp_result += "compare attr [ ERR ]: " + name + " [drop `void` comparison which is '" + - adapter.get_type_info().name + "']"; - } -} } // namespace detail @@ -937,3 +946,42 @@ Comparator::Result compare(ngraph::Node* node1, ngraph::Node* node2, Comparator: } } // namespace attributes + +AccuracyCheckResult accuracy_check(const std::shared_ptr& ref_function, + const std::shared_ptr& cur_function) { + if (ref_function->is_dynamic() || cur_function->is_dynamic()) { + return AccuracyCheckResult{true, ""}; + } + try { + IE_ASSERT(ref_function->get_parameters().size() == cur_function->get_parameters().size()); + + std::map, ov::Tensor> ref_input_data; + std::map, ov::Tensor> cur_input_data; + for (size_t i = 0; i < ref_function->get_parameters().size(); i++) { + const auto &tensor = ov::test::utils::create_and_fill_tensor( + ref_function->get_parameters()[i]->get_element_type(), + ref_function->get_parameters()[i]->get_shape()); + ref_input_data[ref_function->get_parameters()[i]] = tensor; + cur_input_data[cur_function->get_parameters()[i]] = tensor; + } + + auto ref_outputs = ngraph::helpers::interpretFunction(ref_function, ref_input_data); + auto outputs = ngraph::helpers::interpretFunction(cur_function, cur_input_data); + + IE_ASSERT(ref_outputs.size() == outputs.size()); + + for (int i = 0; i < ref_outputs.size(); i++) { + ov::test::utils::compare(ref_outputs[i], outputs[i], + std::numeric_limits::max(), + std::numeric_limits::max()); + } + } + catch (const std::runtime_error &re) { + return AccuracyCheckResult{false, re.what()}; + } catch (const std::exception &ex) { + return AccuracyCheckResult{false, ex.what()}; + } catch (...) { + return AccuracyCheckResult{false, ""}; + } + return AccuracyCheckResult{true, ""}; +} diff --git a/src/tests/ie_test_utils/common_test_utils/graph_comparator.hpp b/src/tests/ie_test_utils/common_test_utils/graph_comparator.hpp index 9cdfe1bf3af..c96753ba65a 100644 --- a/src/tests/ie_test_utils/common_test_utils/graph_comparator.hpp +++ b/src/tests/ie_test_utils/common_test_utils/graph_comparator.hpp @@ -20,12 +20,14 @@ class FunctionsComparator { public: enum CmpValues { NONE = 0, - CONST_VALUES = 1 << 0, - NAMES = 1 << 1, - RUNTIME_KEYS = 1 << 2, - PRECISIONS = 1 << 3, - ATTRIBUTES = 1 << 4, - TENSOR_NAMES = 1 << 5, + NODES = 1 << 0, + CONST_VALUES = 1 << 1, + NAMES = 1 << 2, + RUNTIME_KEYS = 1 << 3, + PRECISIONS = 1 << 4, + ATTRIBUTES = 1 << 5, + TENSOR_NAMES = 1 << 6, + ACCURACY = 1 << 7 }; struct Result { @@ -45,6 +47,7 @@ public: } static FunctionsComparator with_default() noexcept { auto fc = FunctionsComparator::no_default(); + fc.enable(NODES); fc.enable(PRECISIONS); fc.enable(TENSOR_NAMES); return fc; @@ -55,6 +58,11 @@ public: return *this; } + FunctionsComparator& disable(CmpValues f) noexcept { + m_comparison_flags = static_cast(m_comparison_flags & ~f); + return *this; + } + bool should_compare(CmpValues f) const noexcept { return m_comparison_flags & f; } @@ -85,6 +93,7 @@ inline std::pair compare_functions(const std::shared_ptr compare_functions(const std::shared_ptr& f); -void set_tensor_name(ngraph::Output output, const std::string& name); - -void set_tensor_names(ngraph::Output output, const std::unordered_set& names); - namespace ngraph { namespace pass { class InjectionPass : public ov::pass::ModelPass { @@ -188,8 +193,8 @@ public: for (auto output : node->outputs()) { const auto& tensor_names = output.get_names(); if (std::any_of(tensor_names.begin(), tensor_names.end(), [&](const std::string& name) { - return unique_tensor_names.count(name); - })) { + return unique_tensor_names.count(name); + })) { std::stringstream ss; ss << "Node: " << node->get_type_info() << " with name " << node->get_friendly_name() << " "; ss << "has non unique tensor name."; @@ -478,9 +483,9 @@ public: void on_adapter(const std::string& name, ngraph::ValueAccessor& adapter) override; #define ON_ADAPTER(TYPE) \ - void on_adapter(const std::string& name, ngraph::ValueAccessor& adapter) override { \ - insert(name, adapter.get()); \ - } +void on_adapter(const std::string& name, ngraph::ValueAccessor& adapter) override { \ +insert(name, adapter.get()); \ +} ON_ADAPTER(bool) ON_ADAPTER(std::string) @@ -711,37 +716,37 @@ struct Equal> { } switch (lhs_t) { - case ngraph::element::Type_t::u1: { - const auto lhs_v = static_cast(lhs->get_data_ptr()); - const auto rhs_v = static_cast(rhs->get_data_ptr()); - const auto lhs_bit_size = shape_size(lhs->get_shape()); - const auto rhs_bit_size = shape_size(rhs->get_shape()); - return Equal::equal_value(lhs_v, rhs_v, lhs_bit_size, rhs_bit_size); - } - case ngraph::element::Type_t::bf16: { - auto lhs_v = lhs->cast_vector(); - auto rhs_v = rhs->cast_vector(); - return Equal>::equal_value(lhs_v, rhs_v); - break; - } - case ngraph::element::Type_t::f16: { - const auto& lhs_v = lhs->cast_vector(); - const auto& rhs_v = rhs->cast_vector(); - return Equal>::equal_value(lhs_v, rhs_v); - break; - } - case ngraph::element::Type_t::f32: { - const auto& lhs_v = lhs->cast_vector(); - const auto& rhs_v = rhs->cast_vector(); - return Equal>::equal_value(lhs_v, rhs_v); - break; - } - default: { - const auto& lhs_v = lhs->cast_vector(); - const auto& rhs_v = rhs->cast_vector(); - return Equal>::equal_value(lhs_v, rhs_v); - break; - } + case ngraph::element::Type_t::u1: { + const auto lhs_v = static_cast(lhs->get_data_ptr()); + const auto rhs_v = static_cast(rhs->get_data_ptr()); + const auto lhs_bit_size = shape_size(lhs->get_shape()); + const auto rhs_bit_size = shape_size(rhs->get_shape()); + return Equal::equal_value(lhs_v, rhs_v, lhs_bit_size, rhs_bit_size); + } + case ngraph::element::Type_t::bf16: { + auto lhs_v = lhs->cast_vector(); + auto rhs_v = rhs->cast_vector(); + return Equal>::equal_value(lhs_v, rhs_v); + break; + } + case ngraph::element::Type_t::f16: { + const auto& lhs_v = lhs->cast_vector(); + const auto& rhs_v = rhs->cast_vector(); + return Equal>::equal_value(lhs_v, rhs_v); + break; + } + case ngraph::element::Type_t::f32: { + const auto& lhs_v = lhs->cast_vector(); + const auto& rhs_v = rhs->cast_vector(); + return Equal>::equal_value(lhs_v, rhs_v); + break; + } + default: { + const auto& lhs_v = lhs->cast_vector(); + const auto& rhs_v = rhs->cast_vector(); + return Equal>::equal_value(lhs_v, rhs_v); + break; + } } return false; } @@ -862,18 +867,18 @@ struct Get, void> { class ReadAndCompareAttributes : public ngraph::AttributeVisitor { public: ReadAndCompareAttributes(const ReadAndStoreAttributes& ref, Comparator::CmpValues check_flags) - : m_attr_ref(ref), - m_cmp_result{ref.read_result()}, - m_check_flags(check_flags) {} + : m_attr_ref(ref), + m_cmp_result{ref.read_result()}, + m_check_flags(check_flags) {} void on_adapter(const std::string& name, ngraph::ValueAccessor& adapter) override { verify_others(name, adapter); } #define ON_ADAPTER(TYPE) \ - void on_adapter(const std::string& name, ngraph::ValueAccessor& adapter) override { \ - verify(name, adapter.get()); \ - } +void on_adapter(const std::string& name, ngraph::ValueAccessor& adapter) override { \ +verify(name, adapter.get()); \ +} ON_ADAPTER(bool) ON_ADAPTER(std::string) @@ -981,3 +986,11 @@ private: Comparator::Result compare(ngraph::Node* node1, ngraph::Node* node2, Comparator::CmpValues comparition_flags); } // namespace attributes + +struct AccuracyCheckResult { + bool status; + std::string message; +}; + +AccuracyCheckResult accuracy_check(const std::shared_ptr& ref_function, + const std::shared_ptr& cur_function); diff --git a/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.cpp b/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.cpp index 28e7959f65d..b2b2fea8f08 100644 --- a/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.cpp +++ b/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.cpp @@ -2,248 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include #include "ngraph_test_utils.hpp" -#include - -namespace { - -template -static void Compare(const T_NGRAPH *expected, const T_IE *actual, std::size_t size, float threshold, float abs_threshold = -1.f) { - for (std::size_t i = 0; i < size; ++i) { - const T_NGRAPH &ref = expected[i]; - const auto &res = actual[i]; - const auto absoluteDifference = CommonTestUtils::ie_abs(res - ref); - if (abs_threshold > 0.f && absoluteDifference > abs_threshold) { - IE_THROW() << "Absolute comparison of values expected: " << std::to_string(ref) << " and actual: " << std::to_string(res) - << " at index " << i << " with absolute threshold " << abs_threshold - << " failed"; - } - if (absoluteDifference <= threshold) { - continue; - } - double max; - if (sizeof(T_IE) < sizeof(T_NGRAPH)) { - max = std::max(CommonTestUtils::ie_abs(T_NGRAPH(res)), CommonTestUtils::ie_abs(ref)); - } else { - max = std::max(CommonTestUtils::ie_abs(res), CommonTestUtils::ie_abs(T_IE(ref))); - } - double diff = static_cast(absoluteDifference) / max; - if (max == 0 || (diff > static_cast(threshold)) || - (std::isnan(static_cast(res)) ^ std::isnan(static_cast(ref)))) { - IE_THROW() << "Relative comparison of values expected: " << std::to_string(ref) << " and actual: " << std::to_string(res) - << " at index " << i << " with threshold " << threshold - << " failed"; - } - } -} - -template -void callCompare(const std::pair> &expected, - const T_IE* actualBuffer, size_t size, float threshold, float abs_threshold) { - auto expectedBuffer = expected.second.data(); - switch (expected.first) { - case ngraph::element::Type_t::i64: - Compare(reinterpret_cast(expectedBuffer), - actualBuffer, size, threshold, abs_threshold); - break; - case ngraph::element::Type_t::i32: - Compare(reinterpret_cast(expectedBuffer), - actualBuffer, size, threshold, abs_threshold); - break; - case ngraph::element::Type_t::i16: - Compare(reinterpret_cast(expectedBuffer), - actualBuffer, size, threshold, abs_threshold); - break; - case ngraph::element::Type_t::i8: - Compare(reinterpret_cast(expectedBuffer), - actualBuffer, size, threshold, abs_threshold); - break; - case ngraph::element::Type_t::u64: - Compare(reinterpret_cast(expectedBuffer), - actualBuffer, size, threshold, abs_threshold); - break; - case ngraph::element::Type_t::u32: - Compare(reinterpret_cast(expectedBuffer), - actualBuffer, size, threshold, abs_threshold); - break; - case ngraph::element::Type_t::u16: - Compare(reinterpret_cast(expectedBuffer), - actualBuffer, size, threshold, abs_threshold); - break; - case ngraph::element::Type_t::boolean: - case ngraph::element::Type_t::u8: - Compare(reinterpret_cast(expectedBuffer), - actualBuffer, size, threshold, abs_threshold); - break; - case ngraph::element::Type_t::f64: - Compare(reinterpret_cast(expectedBuffer), - actualBuffer, size, threshold, abs_threshold); - break; - case ngraph::element::Type_t::f32: - Compare(reinterpret_cast(expectedBuffer), - actualBuffer, size, threshold, abs_threshold); - break; - case ngraph::element::Type_t::f16: - Compare(reinterpret_cast(expectedBuffer), - actualBuffer, size, threshold, abs_threshold); - break; - case ngraph::element::Type_t::bf16: - Compare(reinterpret_cast(expectedBuffer), - actualBuffer, size, threshold, abs_threshold); - break; -// case ngraph::element::Type_t::i4: { -// auto expectedOut = ngraph::helpers::convertOutputPrecision( -// expected.second, -// expected.first, -// ngraph::element::Type_t::i8, -// size); -// Compare(reinterpret_cast(expectedOut.data()), -// actualBuffer, size, threshold, abs_threshold); -// break; -// } -// case ngraph::element::Type_t::u4: { -// auto expectedOut = ngraph::helpers::convertOutputPrecision( -// expected.second, -// expected.first, -// ngraph::element::Type_t::u8, -// size); -// Compare(reinterpret_cast(expectedOut.data()), -// actualBuffer, size, threshold, abs_threshold); -// break; -// } - case ngraph::element::Type_t::dynamic: - case ngraph::element::Type_t::undefined: - Compare(reinterpret_cast(expectedBuffer), actualBuffer, size, threshold, abs_threshold); - break; - default: FAIL() << "Comparator for " << expected.first << " precision isn't supported"; - } - return; -} - - -void Compare(const std::pair> &expected, - const InferenceEngine::Blob::Ptr &actual, - float threshold, - float abs_threshold) { - const auto &precision = actual->getTensorDesc().getPrecision(); - auto k = static_cast(expected.first.size()) / precision.size(); - // W/A for int4, uint4 - if (expected.first == ngraph::element::Type_t::u4 || expected.first == ngraph::element::Type_t::i4) { - k /= 2; - } else if (expected.first == ngraph::element::Type_t::undefined || expected.first == ngraph::element::Type_t::dynamic) { - k = 1; - } - ASSERT_EQ(expected.second.size(), actual->byteSize() * k); - - auto memory = InferenceEngine::as(actual); - IE_ASSERT(memory); - const auto lockedMemory = memory->wmap(); - const auto actualBuffer = lockedMemory.as(); - - const auto &size = actual->size(); - switch (precision) { - case InferenceEngine::Precision::FP32: - callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); - break; - case InferenceEngine::Precision::I32: - callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); - break; - case InferenceEngine::Precision::I64: - callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); - break; - case InferenceEngine::Precision::I8: - callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); - break; - case InferenceEngine::Precision::U16: - callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); - break; - case InferenceEngine::Precision::I16: - callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); - break; - case InferenceEngine::Precision::BOOL: - case InferenceEngine::Precision::U8: - callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); - break; - case InferenceEngine::Precision::U64: - callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); - break; - case InferenceEngine::Precision::BF16: - callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); - break; - case InferenceEngine::Precision::FP16: - callCompare(expected, reinterpret_cast(actualBuffer), size, threshold, abs_threshold); - break; - default: - FAIL() << "Comparator for " << precision << " precision isn't supported"; - } -} - -void Compare(const std::vector>> &expectedOutputs, - const std::vector &actualOutputs, - float threshold, float abs_threshold) { - for (std::size_t outputIndex = 0; outputIndex < expectedOutputs.size(); ++outputIndex) { - const auto &expected = expectedOutputs[outputIndex]; - const auto &actual = actualOutputs[outputIndex]; - Compare(expected, actual, threshold, abs_threshold); - } -} -} // namespace - -void TransformationTestsF::accuracy_check(std::shared_ptr ref_function, - std::shared_ptr cur_function) { - try { - if (ref_function->is_dynamic() || cur_function->is_dynamic()) { - return; - } - std::vector> input_data; - ngraph::element::TypeVector types; - for (auto param : ref_function->get_parameters()) { - types.push_back(param->get_element_type()); - - auto layout = InferenceEngine::Layout::ANY; - if (ov::is_scalar(param->get_shape())) { - layout = InferenceEngine::Layout::SCALAR; - } - InferenceEngine::TensorDesc td(InferenceEngine::Precision::FP32, param->get_shape(), layout); - const auto &input = FuncTestUtils::createAndFillBlob(td); - const auto &input_size = input->byteSize(); - - std::vector data; - data.resize(input_size); - - auto memory = InferenceEngine::as(input); - IE_ASSERT(memory); - - const auto lockedMemory = memory->wmap(); - const auto buffer = lockedMemory.as(); - std::copy(buffer, buffer + input_size, data.data()); - - input_data.push_back(std::move(data)); - } - - auto ref_outputs = ngraph::helpers::interpreterFunction(ref_function, input_data, types); - auto outputs = ngraph::helpers::interpreterFunction(cur_function, input_data, types); - - IE_ASSERT(ref_outputs.size() == outputs.size()); - - for (size_t i = 0; i < ref_outputs.size(); ++i) { - IE_ASSERT(ref_outputs[i].second.size() == outputs[i].second.size()); - auto * ref = reinterpret_cast(ref_outputs[i].second.data()); - auto * out = reinterpret_cast(outputs[i].second.data()); - size_t size = ref_outputs[i].second.size() / sizeof(float); - IE_ASSERT(size > 0); - Compare(ref, out, size, 1e-5); - } - } - catch (const std::runtime_error &re) { - GTEST_FATAL_FAILURE_(re.what()); - } catch (const std::exception &ex) { - GTEST_FATAL_FAILURE_(ex.what()); - } catch (...) { - GTEST_FATAL_FAILURE_("Unknown failure occurred."); - } -} void init_unique_names(std::shared_ptr f, const std::shared_ptr& unh) { ngraph::pass::Manager manager; diff --git a/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.hpp b/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.hpp index 549ada20f7f..c79898e1930 100644 --- a/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.hpp +++ b/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.hpp @@ -32,6 +32,7 @@ class TransformationTestsF : public CommonTestUtils::TestsCommon { public: TransformationTestsF() : comparator(FunctionsComparator::no_default()) { m_unh = std::make_shared(); + comparator.enable(FunctionsComparator::CmpValues::NODES); comparator.enable(FunctionsComparator::CmpValues::PRECISIONS); comparator.enable(FunctionsComparator::CmpValues::RUNTIME_KEYS); // TODO: enable attributes and constant values comparison by default XXX-68694 @@ -56,12 +57,16 @@ public: ASSERT_NO_THROW(check_rt_info(function)); } + if (comparator.should_compare(FunctionsComparator::ACCURACY)) { + auto acc_comparator = FunctionsComparator::no_default(); + acc_comparator.enable(FunctionsComparator::CmpValues::ACCURACY); + auto res = acc_comparator.compare(function, cloned_function); + ASSERT_TRUE(res.valid) << res.message; + comparator.disable(FunctionsComparator::CmpValues::ACCURACY); + } + auto res = comparator.compare(function, function_ref); ASSERT_TRUE(res.valid) << res.message; - - if (m_enable_accuracy_check) { - accuracy_check(cloned_function, function); - } } // TODO: this is temporary solution to disable rt info checks that must be applied by default @@ -74,12 +79,6 @@ public: m_soft_names_comparison = true; } - void enable_accuracy_check() { - m_enable_accuracy_check = true; - } - - void accuracy_check(std::shared_ptr ref_function, std::shared_ptr cur_function); - std::shared_ptr function, function_ref; ngraph::pass::Manager manager; FunctionsComparator comparator; @@ -88,7 +87,6 @@ private: std::shared_ptr m_unh; bool m_disable_rt_info_check{false}; bool m_soft_names_comparison{true}; - bool m_enable_accuracy_check{false}; }; void init_unique_names(std::shared_ptr f, const std::shared_ptr& unh); From e507b630ebd66440a1fb02b8820f1dea5a00f655 Mon Sep 17 00:00:00 2001 From: Egor Shulman Date: Wed, 30 Mar 2022 18:26:35 +0300 Subject: [PATCH 05/39] Use CC in CPUSpecificTransform (#11231) --- .../align_matmul_input_ranks.cpp | 4 +- .../convert_broadcast_to_tiles.cpp | 5 ++- .../convert_matmul_to_fc.cpp | 5 ++- .../convert_tile_to_seq_tiles.cpp | 5 ++- .../convert_to_cpu_specific_opset.hpp | 3 ++ .../convert_to_leaky_relu.cpp | 5 ++- .../convert_to_power_static.cpp | 5 ++- .../convert_to_swish_cpu.cpp | 5 ++- .../ngraph_transformations/fc_bias_fusion.cpp | 5 ++- .../src/ngraph_transformations/itt.hpp | 43 +++++++++++++++++++ .../move_eltwise_up_data_movement.cpp | 5 ++- .../op/fully_connected.cpp | 4 ++ .../ngraph_transformations/op/leaky_relu.cpp | 4 ++ .../op/power_static.cpp | 4 ++ .../ngraph_transformations/op/swish_cpu.cpp | 4 ++ .../reshape_fc_fusion.cpp | 5 ++- .../ngraph_transformations/reshape_prelu.cpp | 5 ++- .../rnn_sequences_optimization.cpp | 11 +++-- .../snippets_mark_skipped.cpp | 3 ++ .../swap_convert_transpose.cpp | 5 ++- 20 files changed, 120 insertions(+), 15 deletions(-) create mode 100644 src/plugins/intel_cpu/src/ngraph_transformations/itt.hpp diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/align_matmul_input_ranks.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/align_matmul_input_ranks.cpp index 6f3c4c45cb3..517cf337679 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/align_matmul_input_ranks.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/align_matmul_input_ranks.cpp @@ -12,8 +12,10 @@ #include #include +#include "itt.hpp" ov::intel_cpu::AlignMatMulInputRanks::AlignMatMulInputRanks() { + MATCHER_SCOPE(AlignMatMulInputRanks); ngraph::OutputVector twoInputs = { ngraph::pattern::any_input(ngraph::pattern::has_static_rank()), ngraph::pattern::any_input(ngraph::pattern::has_static_rank()) @@ -135,6 +137,6 @@ ov::intel_cpu::AlignMatMulInputRanks::AlignMatMulInputRanks() { return true; }; - auto m = std::make_shared(matmulPattern, "AlignMatMulInputRanks"); + auto m = std::make_shared(matmulPattern, matcher_name); this->register_matcher(m, callback); } diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_broadcast_to_tiles.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/convert_broadcast_to_tiles.cpp index 08036e188ee..69dfeed6a3d 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/convert_broadcast_to_tiles.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/convert_broadcast_to_tiles.cpp @@ -8,7 +8,10 @@ #include #include +#include "itt.hpp" + ov::intel_cpu::ConvertBroadcastToTiles::ConvertBroadcastToTiles() { + MATCHER_SCOPE(ConvertBroadcastToTiles); auto broadcast = ngraph::pattern::wrap_type(); ngraph::matcher_pass_callback callback = [this](ngraph::pattern::Matcher& m) { @@ -91,6 +94,6 @@ ov::intel_cpu::ConvertBroadcastToTiles::ConvertBroadcastToTiles() { return true; }; - auto m = std::make_shared(broadcast, "ConvertBroadcastToTiles"); + auto m = std::make_shared(broadcast, matcher_name); this->register_matcher(m, callback); } diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_matmul_to_fc.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/convert_matmul_to_fc.cpp index c9369fefad1..d176dd857ff 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/convert_matmul_to_fc.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/convert_matmul_to_fc.cpp @@ -9,7 +9,10 @@ #include #include +#include "itt.hpp" + ov::intel_cpu::ConvertMatMulToFC::ConvertMatMulToFC() { + MATCHER_SCOPE(ConvertMatMulToFC); auto activations_m = ngraph::pattern::any_input(ngraph::pattern::has_static_rank()); auto weights_m = ngraph::pattern::wrap_type(); auto matmul_m = ngraph::pattern::wrap_type({ activations_m, weights_m }, ngraph::pattern::has_static_rank()); @@ -162,6 +165,6 @@ ov::intel_cpu::ConvertMatMulToFC::ConvertMatMulToFC() { return true; }; - auto m = std::make_shared(matmul_m, "ConvertMatMulToFC"); + auto m = std::make_shared(matmul_m, matcher_name); this->register_matcher(m, callback); } diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_tile_to_seq_tiles.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/convert_tile_to_seq_tiles.cpp index a21fcafd24d..fa0609e528a 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/convert_tile_to_seq_tiles.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/convert_tile_to_seq_tiles.cpp @@ -12,7 +12,10 @@ #include +#include "itt.hpp" + ov::intel_cpu::ConvertTileToSeqTiles::ConvertTileToSeqTiles() { + MATCHER_SCOPE(ConvertTileToSeqTiles); auto tile = ngraph::pattern::wrap_type({ngraph::pattern::any_input(ngraph::pattern::has_static_rank()), ngraph::pattern::wrap_type()}); @@ -88,6 +91,6 @@ ov::intel_cpu::ConvertTileToSeqTiles::ConvertTileToSeqTiles() { return true; }; - auto m = std::make_shared(tile, "ConvertTileToSeqTiles"); + auto m = std::make_shared(tile, matcher_name); this->register_matcher(m, callback); } diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_cpu_specific_opset.hpp b/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_cpu_specific_opset.hpp index 7da63c8f5be..9a6427fdf43 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_cpu_specific_opset.hpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_cpu_specific_opset.hpp @@ -20,10 +20,13 @@ #include "rnn_sequences_optimization.hpp" #include "transformations/common_optimizations/reshape_sequence_fusion.hpp" +#include "itt.hpp" + namespace ov { namespace intel_cpu { inline void ConvertToCPUSpecificOpset(std::shared_ptr &nGraphFunc) { + RUN_ON_FUNCTION_SCOPE(ConvertToCPUSpecificOpset); ngraph::pass::Manager manager; manager.register_pass(); manager.register_pass(); diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_leaky_relu.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_leaky_relu.cpp index d46fbc3d896..dc1f73bd581 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_leaky_relu.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_leaky_relu.cpp @@ -9,7 +9,10 @@ #include #include "op/leaky_relu.hpp" +#include "itt.hpp" + ov::intel_cpu::ConvertToLeakyRelu::ConvertToLeakyRelu() { + MATCHER_SCOPE(ConvertToLeakyRelu); auto input = ngraph::pattern::any_input(); auto slope_constant = ngraph::pattern::wrap_type(); auto prelu = ngraph::pattern::wrap_type({ input, slope_constant }); @@ -32,6 +35,6 @@ ov::intel_cpu::ConvertToLeakyRelu::ConvertToLeakyRelu() { return false; }; - auto m = std::make_shared(prelu, "ConvertToLeakyRelu"); + auto m = std::make_shared(prelu, matcher_name); this->register_matcher(m, callback); } diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_power_static.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_power_static.cpp index 3a98458ad31..a090bd00fa3 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_power_static.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_power_static.cpp @@ -14,6 +14,8 @@ #include "op/fully_connected.hpp" #include "utils/general_utils.h" +#include "itt.hpp" + namespace { int getConstPort(const std::shared_ptr &node) { @@ -97,6 +99,7 @@ std::shared_ptr convert(const std::shared_ptr &node) { } // namespace ov::intel_cpu::ConvertToPowerStatic::ConvertToPowerStatic() { + MATCHER_SCOPE(ConvertToPowerStatic); ngraph::OutputVector twoInputs = {ngraph::pattern::any_input(ngraph::pattern::has_static_rank()), ngraph::pattern::any_input(ngraph::pattern::has_static_rank())}; auto power = ngraph::pattern::wrap_type(twoInputs); @@ -134,6 +137,6 @@ ov::intel_cpu::ConvertToPowerStatic::ConvertToPowerStatic() { return true; }; - auto m = std::make_shared(candidate, "ConvertToPowerStatic"); + auto m = std::make_shared(candidate, matcher_name); this->register_matcher(m, callback); } diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_swish_cpu.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_swish_cpu.cpp index 7e7b24a29a4..f274efad680 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_swish_cpu.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/convert_to_swish_cpu.cpp @@ -9,7 +9,10 @@ #include #include "op/swish_cpu.hpp" +#include "itt.hpp" + ov::intel_cpu::ConvertToSwishCPU::ConvertToSwishCPU() { + MATCHER_SCOPE(ConvertToSwishCPU); auto swish = ngraph::pattern::wrap_type(); ngraph::matcher_pass_callback callback = [](ngraph::pattern::Matcher& m) { @@ -34,6 +37,6 @@ ov::intel_cpu::ConvertToSwishCPU::ConvertToSwishCPU() { return true; }; - auto m = std::make_shared(swish, "ConvertToSwishCPU"); + auto m = std::make_shared(swish, matcher_name); this->register_matcher(m, callback); } diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/fc_bias_fusion.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/fc_bias_fusion.cpp index 5524d839718..77ffe5632b1 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/fc_bias_fusion.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/fc_bias_fusion.cpp @@ -11,7 +11,10 @@ #include "transformations/utils/utils.hpp" +#include "itt.hpp" + ov::intel_cpu::FullyConnectedBiasFusion::FullyConnectedBiasFusion() { + MATCHER_SCOPE(FullyConnectedBiasFusion); auto input = ngraph::pattern::any_input(); auto weights = ngraph::pattern::any_input(ngraph::pattern::has_static_shape()); auto m_fc = ngraph::pattern::wrap_type({ input, weights }, [](ngraph::Output output) { @@ -68,6 +71,6 @@ ov::intel_cpu::FullyConnectedBiasFusion::FullyConnectedBiasFusion() { return true; }; - auto m = std::make_shared(m_add, "FullyConnectedBiasFusion"); + auto m = std::make_shared(m_add, matcher_name); this->register_matcher(m, callback); } diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/itt.hpp b/src/plugins/intel_cpu/src/ngraph_transformations/itt.hpp new file mode 100644 index 00000000000..c54f290b857 --- /dev/null +++ b/src/plugins/intel_cpu/src/ngraph_transformations/itt.hpp @@ -0,0 +1,43 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief Defines openvino domains for tracing + * @file itt.hpp + */ + +#pragma once + +#include + +namespace ov { +namespace intel_cpu { +namespace itt { +namespace domains { + OV_ITT_DOMAIN(CPUSpecificTransform); +} // namespace domains +} // namespace itt +} // namespace intel_cpu +} // namespace ov + +OV_CC_DOMAINS(internal_op); + +/* + * RUN_ON_FUNCTION_SCOPE macro allows to disable the run_on_function pass + * MATCHER_SCOPE macro allows to disable the MatcherPass if matcher isn't applied + * INTERNAL_OP_SCOPE macro allows to disable parts of internal nGraph operations if they are not used + */ +#if defined(SELECTIVE_BUILD_ANALYZER) + +#define INTERNAL_OP_SCOPE(region) OV_SCOPE(internal_op, region) + +#elif defined(SELECTIVE_BUILD) + +#define INTERNAL_OP_SCOPE(region) MATCHER_SCOPE_(internal_op, region) + +#else + +#define INTERNAL_OP_SCOPE(region) + +#endif diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/move_eltwise_up_data_movement.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/move_eltwise_up_data_movement.cpp index 72a1404ff57..b5a733c623b 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/move_eltwise_up_data_movement.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/move_eltwise_up_data_movement.cpp @@ -12,6 +12,8 @@ #include #include +#include "itt.hpp" + namespace { bool is_data_movement_operation(const std::shared_ptr& node) { @@ -38,6 +40,7 @@ namespace { } // namespace ov::intel_cpu::MoveEltwiseUpThroughDataMov::MoveEltwiseUpThroughDataMov() { + MATCHER_SCOPE(MoveEltwiseUpThroughDataMov); auto eltwise_pattern = ngraph::pattern::wrap_type(ngraph::pattern::has_static_rank()); @@ -108,6 +111,6 @@ ov::intel_cpu::MoveEltwiseUpThroughDataMov::MoveEltwiseUpThroughDataMov() { return true; }; - auto m = std::make_shared(eltwise_pattern, "MoveEltwiseUpThroughDataMov"); + auto m = std::make_shared(eltwise_pattern, matcher_name); register_matcher(m, callback); } diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/fully_connected.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/op/fully_connected.cpp index 936e9beb2a6..c9da277bcc2 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/op/fully_connected.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/op/fully_connected.cpp @@ -3,6 +3,7 @@ // #include "fully_connected.hpp" +#include "../itt.hpp" ov::intel_cpu::FullyConnectedNode::FullyConnectedNode(const ngraph::Output& A, const ngraph::Output& B, @@ -22,6 +23,7 @@ ov::intel_cpu::FullyConnectedNode::FullyConnectedNode(const ngraph::Output } std::shared_ptr ov::intel_cpu::FullyConnectedNode::clone_with_new_inputs(const ngraph::OutputVector& new_args) const { + INTERNAL_OP_SCOPE(FullyConnectedNode_clone_with_new_inputs); check_new_args_count(this, new_args); if (new_args.size() == 2) { return std::make_shared(new_args.at(0), new_args.at(1), m_output_rank, m_output_type); @@ -33,6 +35,7 @@ std::shared_ptr ov::intel_cpu::FullyConnectedNode::clone_with_new_ } void ov::intel_cpu::FullyConnectedNode::validate_and_infer_types() { + INTERNAL_OP_SCOPE(FullyConnectedNode_validate_and_infer_types); const auto input_size = get_input_size(); NODE_VALIDATION_CHECK(this, input_size == 2 || input_size == 3, @@ -94,6 +97,7 @@ void ov::intel_cpu::FullyConnectedNode::validate_and_infer_types() { } bool ov::intel_cpu::FullyConnectedNode::visit_attributes(ngraph::AttributeVisitor &visitor) { + INTERNAL_OP_SCOPE(FullyConnectedNode_visit_attributes); visitor.on_attribute("out-rank", m_output_rank); visitor.on_attribute("out-type", m_output_type); return true; diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/leaky_relu.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/op/leaky_relu.cpp index 4f2899d7c3c..e1e63b456b9 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/op/leaky_relu.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/op/leaky_relu.cpp @@ -3,6 +3,7 @@ // #include "leaky_relu.hpp" +#include "../itt.hpp" ov::intel_cpu::LeakyReluNode::LeakyReluNode(const ngraph::Output &data, const float &negative_slope, @@ -12,11 +13,13 @@ ov::intel_cpu::LeakyReluNode::LeakyReluNode(const ngraph::Output & } std::shared_ptr ov::intel_cpu::LeakyReluNode::clone_with_new_inputs(const ngraph::OutputVector& new_args) const { + INTERNAL_OP_SCOPE(LeakyReluNode_clone_with_new_inputs); check_new_args_count(this, new_args); return std::make_shared(new_args.at(0), m_negative_slope, m_output_type); } void ov::intel_cpu::LeakyReluNode::validate_and_infer_types() { + INTERNAL_OP_SCOPE(LeakyReluNode_validate_and_infer_types); set_output_type( 0, m_output_type == ngraph::element::undefined ? get_input_element_type(0) : m_output_type, @@ -24,6 +27,7 @@ void ov::intel_cpu::LeakyReluNode::validate_and_infer_types() { } bool ov::intel_cpu::LeakyReluNode::visit_attributes(ngraph::AttributeVisitor &visitor) { + INTERNAL_OP_SCOPE(LeakyReluNode_visit_attributes); visitor.on_attribute("negative_slope", m_negative_slope); visitor.on_attribute("out-type", m_output_type); return true; diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/power_static.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/op/power_static.cpp index cddc10fa06e..976066a48e4 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/op/power_static.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/op/power_static.cpp @@ -3,6 +3,7 @@ // #include "power_static.hpp" +#include "../itt.hpp" ov::intel_cpu::PowerStaticNode::PowerStaticNode(const ngraph::Output &data, const float &power, @@ -14,6 +15,7 @@ ov::intel_cpu::PowerStaticNode::PowerStaticNode(const ngraph::Output &data } std::shared_ptr ov::intel_cpu::PowerStaticNode::clone_with_new_inputs(const ngraph::OutputVector &new_args) const { + INTERNAL_OP_SCOPE(PowerStaticNode_clone_with_new_inputs); if (new_args.size() != 1) { throw ngraph::ngraph_error("Incorrect number of new arguments"); } @@ -22,10 +24,12 @@ std::shared_ptr ov::intel_cpu::PowerStaticNode::clone_with_new_inp } void ov::intel_cpu::PowerStaticNode::validate_and_infer_types() { + INTERNAL_OP_SCOPE(PowerStaticNode_validate_and_infer_types); set_output_type(0, m_output_type == ngraph::element::undefined ? get_input_element_type(0) : m_output_type, get_input_partial_shape(0)); } bool ov::intel_cpu::PowerStaticNode::visit_attributes(ngraph::AttributeVisitor &visitor) { + INTERNAL_OP_SCOPE(PowerStaticNode_visit_attributes); visitor.on_attribute("scale", scale); visitor.on_attribute("power", power); visitor.on_attribute("shift", shift); diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/op/swish_cpu.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/op/swish_cpu.cpp index f0628ad252c..e4cc5de6810 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/op/swish_cpu.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/op/swish_cpu.cpp @@ -3,6 +3,7 @@ // #include "swish_cpu.hpp" +#include "../itt.hpp" ov::intel_cpu::SwishNode::SwishNode(const ngraph::Output & input, const float alpha) : Op({input}), m_alpha(alpha) { @@ -10,16 +11,19 @@ ov::intel_cpu::SwishNode::SwishNode(const ngraph::Output & input, } std::shared_ptr ov::intel_cpu::SwishNode::clone_with_new_inputs(const ngraph::OutputVector& new_args) const { + INTERNAL_OP_SCOPE(SwishNode_clone_with_new_inputs); check_new_args_count(this, new_args); return std::make_shared(new_args.at(0), m_alpha); } bool ov::intel_cpu::SwishNode::visit_attributes(ngraph::AttributeVisitor& visitor) { + INTERNAL_OP_SCOPE(SwishNode_visit_attributes); visitor.on_attribute("alpha", m_alpha); return true; } void ov::intel_cpu::SwishNode::validate_and_infer_types() { + INTERNAL_OP_SCOPE(SwishNode_validate_and_infer_types); set_output_type(0, get_input_element_type(0), get_input_partial_shape(0)); } diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/reshape_fc_fusion.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/reshape_fc_fusion.cpp index 7b43b43e25c..e2cb71f0890 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/reshape_fc_fusion.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/reshape_fc_fusion.cpp @@ -10,7 +10,10 @@ #include #include +#include "itt.hpp" + ov::intel_cpu::ReshapeFullyConnectedFusion::ReshapeFullyConnectedFusion() { + MATCHER_SCOPE(ReshapeFullyConnectedFusion); auto m_reshape = ngraph::pattern::wrap_type({ngraph::pattern::any_input(ov::pass::pattern::has_static_shape()), ngraph::pattern::any_input()}, ngraph::pattern::has_static_shape()); @@ -81,6 +84,6 @@ ov::intel_cpu::ReshapeFullyConnectedFusion::ReshapeFullyConnectedFusion() { return true; }; - auto m = std::make_shared(fcTwoOrThreeInputs, "ReshapeFullyConnectedFusion"); + auto m = std::make_shared(fcTwoOrThreeInputs, matcher_name); register_matcher(m, callback); } diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/reshape_prelu.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/reshape_prelu.cpp index 649052b0614..0ee300ab813 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/reshape_prelu.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/reshape_prelu.cpp @@ -9,7 +9,10 @@ #include #include "transformations/utils/utils.hpp" +#include "itt.hpp" + ov::intel_cpu::ReshapePRelu::ReshapePRelu() { + MATCHER_SCOPE(ReshapeFullyConnectedFusion); auto input_m = ngraph::pattern::any_input(ngraph::pattern::has_static_rank()); auto slope_m = ngraph::pattern::any_input(ngraph::pattern::has_static_rank()); auto prelu_m = ngraph::pattern::wrap_type({ input_m, slope_m }); @@ -49,6 +52,6 @@ ov::intel_cpu::ReshapePRelu::ReshapePRelu() { return true; }; - auto m = std::make_shared(prelu_m, "ReshapePRelu"); + auto m = std::make_shared(prelu_m, matcher_name); this->register_matcher(m, callback); } diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/rnn_sequences_optimization.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/rnn_sequences_optimization.cpp index 6c111fd3d49..bf724a84c54 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/rnn_sequences_optimization.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/rnn_sequences_optimization.cpp @@ -11,6 +11,8 @@ #include #include +#include "itt.hpp" + namespace { int64_t getSeqAxis(const std::shared_ptr& sequenceOp) { // Optimization. @@ -82,6 +84,7 @@ namespace { } // namespace ov::intel_cpu::OptimizeGRUSequenceTransposes::OptimizeGRUSequenceTransposes() { + MATCHER_SCOPE(OptimizeGRUSequenceTransposes); auto gruSequenceNgraph = ngraph::pattern::wrap_type(); ngraph::matcher_pass_callback callback = [](ngraph::pattern::Matcher &m) { @@ -96,11 +99,12 @@ ov::intel_cpu::OptimizeGRUSequenceTransposes::OptimizeGRUSequenceTransposes() { return transform(gruSequence); }; - auto m = std::make_shared(gruSequenceNgraph, "OptimizeGRUSequenceTransposes"); + auto m = std::make_shared(gruSequenceNgraph, matcher_name); this->register_matcher(m, callback); } ov::intel_cpu::OptimizeRNNSequenceTransposes::OptimizeRNNSequenceTransposes() { + MATCHER_SCOPE(OptimizeRNNSequenceTransposes); auto rnnSequenceNgraph = ngraph::pattern::wrap_type(); ngraph::matcher_pass_callback callback = [](ngraph::pattern::Matcher &m) { @@ -115,11 +119,12 @@ ov::intel_cpu::OptimizeRNNSequenceTransposes::OptimizeRNNSequenceTransposes() { return transform(rnnSequence); }; - auto m = std::make_shared(rnnSequenceNgraph, "OptimizeRNNSequenceTransposes"); + auto m = std::make_shared(rnnSequenceNgraph, matcher_name); this->register_matcher(m, callback); } ov::intel_cpu::OptimizeLSTMSequenceTransposes::OptimizeLSTMSequenceTransposes() { + MATCHER_SCOPE(OptimizeLSTMSequenceTransposes); auto lstmSequenceNgraph = ngraph::pattern::wrap_type(); ngraph::matcher_pass_callback callback = [](ngraph::pattern::Matcher &m) { @@ -137,7 +142,7 @@ ov::intel_cpu::OptimizeLSTMSequenceTransposes::OptimizeLSTMSequenceTransposes() return checkSequence(lstmSequence) ? transform(lstmSequence) : false; }; - auto m = std::make_shared(lstmSequenceNgraph, "OptimizeLSTMSequenceTransposes"); + auto m = std::make_shared(lstmSequenceNgraph, matcher_name); this->register_matcher(m, callback); } diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/snippets_mark_skipped.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/snippets_mark_skipped.cpp index 16061da83a4..77d3d634adb 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/snippets_mark_skipped.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/snippets_mark_skipped.cpp @@ -7,6 +7,8 @@ #include #include +#include "itt.hpp" + using namespace ngraph; namespace ov { @@ -297,6 +299,7 @@ void MarkSubgraphOpAsSkipped(const std::shared_ptr &node) { } // namespace bool SnippetsMarkSkipped::run_on_model(const std::shared_ptr &m) { + RUN_ON_MODEL_SCOPE(SnippetsMarkSkipped); for (auto &node : m->get_ordered_ops()) { if (ngraph::op::is_constant(node)) continue; diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/swap_convert_transpose.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/swap_convert_transpose.cpp index 5b7899928ce..69695f4ffe5 100755 --- a/src/plugins/intel_cpu/src/ngraph_transformations/swap_convert_transpose.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/swap_convert_transpose.cpp @@ -8,9 +8,12 @@ #include #include +#include "itt.hpp" + NGRAPH_RTTI_DEFINITION(ov::intel_cpu::SwapConvertTranspose, "SwapConvertTranspose", 0); ov::intel_cpu::SwapConvertTranspose::SwapConvertTranspose() { + MATCHER_SCOPE(SwapConvertTranspose); ngraph::element::TypeVector param_precisions{ ngraph::element::i8, ngraph::element::u8 }; auto input_m = ngraph::pattern::wrap_type(ngraph::pattern::type_matches_any(param_precisions)); auto convert_m = ngraph::pattern::wrap_type({input_m}, ngraph::pattern::type_matches(ngraph::element::f32)); @@ -38,6 +41,6 @@ ov::intel_cpu::SwapConvertTranspose::SwapConvertTranspose() { return true; }; - auto m = std::make_shared(transpose_m, "SwapConvertTranspose"); + auto m = std::make_shared(transpose_m, matcher_name); this->register_matcher(m, callback); } From 1906c27c2d19c2f98cb27ffc4e7dd48ca7b0d354 Mon Sep 17 00:00:00 2001 From: Ilya Sharikov Date: Wed, 30 Mar 2022 18:32:53 +0300 Subject: [PATCH 06/39] Update mo_tool parameter for converter.py (#11319) * Update mo_tool parameter for converter.py * Hot fix --- tests/stress_tests/scripts/get_testdata.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/stress_tests/scripts/get_testdata.py b/tests/stress_tests/scripts/get_testdata.py index 3190f7b19e3..e5e770d5a7d 100755 --- a/tests/stress_tests/scripts/get_testdata.py +++ b/tests/stress_tests/scripts/get_testdata.py @@ -112,6 +112,7 @@ def main(): 'which will be downloaded and converted to IRs via OMZ.') parser.add_argument('--omz_repo', required=False, help='Path to Open Model Zoo (OMZ) repository. It will be used to skip cloning step.') + # TODO: update discretion for --mo_tool. When MO installed to python venv path to MO is redundant parser.add_argument('--mo_tool', type=Path, help='Path to Model Optimizer (MO) runner. Required for OMZ converter.py only.') parser.add_argument('--omz_models_out_dir', type=Path, @@ -144,6 +145,7 @@ def main(): # prepare virtual environment and install requirements python_executable = sys.executable if not args.no_venv: + # TODO: Update paths to requirements, currently paths are wrong Venv = VirtualEnv("./.stress_venv") requirements = [ args.mo_tool.parents[3] / "requirements.txt", @@ -205,15 +207,15 @@ def main(): # convert models to IRs converter_path = omz_path / "tools" / "model_tools" / "converter.py" + # NOTE: remove --precisions if both precisions (FP32 & FP16) required - cmd = '"{executable}" {converter_path} --name {model_name}' \ - ' -p "{executable}"' \ - ' --precisions={precision}' \ - ' --output_dir {irs_dir}' \ - ' --download_dir {models_dir}' \ - ' --mo {mo_tool}'.format(executable=python_executable, converter_path=converter_path, - precision=precision, model_name=model_name, irs_dir=args.omz_irs_out_dir, - models_dir=args.omz_models_out_dir, mo_tool=args.mo_tool) + cmd = f'"{python_executable}" {converter_path} --name {model_name}' \ + f' -p "{python_executable}"' \ + f' --precisions={precision}' \ + f' --output_dir {args.omz_irs_out_dir}' \ + f' --download_dir {args.omz_models_out_dir}' + if args.mo_tool: + cmd += f' --mo {args.mo_tool}' run_in_subprocess(cmd, check_call=not args.skip_omz_errors) for model_rec in model_recs: From a635150b9d2bddb22ba3a9548b96395d2239f8dc Mon Sep 17 00:00:00 2001 From: Mikhail Nosov Date: Wed, 30 Mar 2022 18:56:49 +0300 Subject: [PATCH 07/39] [IE Common] Enable explicit TBlob declaration in all compilers (#11183) * Enable explicit TBlob declaration in all compilers This fixes problems when linking gcc compiled IE with clang compiled applications. Previous to this change, only clang compilers would consider TBlob templated types as declared externally. When *declared* explictly (with the `extern template` syntax), the C++ spec says that any inline methods of the templated class (such as TBlob constructors) should be ignored in favor of the externally instantiated version of that templated type: "An explicit instantiation declaration (an extern template) skips implicit instantiation step: the code that would otherwise cause an implicit instantiation instead uses the explicit instantiation definition provided elsewhere (resulting in link errors if no such instantiation exists)." However, when IE is compiled with gcc, it does not see the explicit `extern template` declarations of TBlob (due to the `#ifdef __clang__` guards in `ie_blob.h`). As an end result, presumably due to link-time-optimizations during IE library compilation(?), none of the TBlob implementations are actually included in the IE dynamic libraries. * Fix warnings for windows * Fix typo --- .ci/azure/linux.yml | 11 ++++++++- src/inference/include/ie/ie_blob.h | 9 ++++--- src/inference/src/ie_blob_common.cpp | 35 +++++++++++++--------------- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/.ci/azure/linux.yml b/.ci/azure/linux.yml index 25880afc586..7cccd692c8f 100644 --- a/.ci/azure/linux.yml +++ b/.ci/azure/linux.yml @@ -109,7 +109,8 @@ jobs: set -e $(REPO_DIR)/install_build_dependencies.sh # Move jdk into contrib - sudo apt --assume-yes install openjdk-11-jdk + # 'clang' compiler is to check that samples can be built using it + sudo apt --assume-yes install openjdk-11-jdk clang # For opencv-python: python3-setuptools and pip upgrade python3 -m pip install --upgrade pip python3 -m pip install -r $(REPO_DIR)/src/bindings/python/src/compatibility/openvino/requirements.txt @@ -223,6 +224,14 @@ jobs: displayName: 'Build cpp samples' continueOnError: false + - script: | + export CC=clang + export CXX=clang++ + $(INSTALL_DIR)/samples/cpp/build_samples.sh -i $(INSTALL_DIR) + workingDirectory: $(BUILD_SAMPLES_DIR) + displayName: 'Build cpp samples - clang' + continueOnError: false + - script: $(INSTALL_DIR)/samples/c/build_samples.sh -i $(INSTALL_DIR) workingDirectory: $(BUILD_SAMPLES_DIR) displayName: 'Build c samples' diff --git a/src/inference/include/ie/ie_blob.h b/src/inference/include/ie/ie_blob.h index 6583c103903..b68a5ed082e 100644 --- a/src/inference/include/ie/ie_blob.h +++ b/src/inference/include/ie/ie_blob.h @@ -599,7 +599,9 @@ public: /** *@brief Virtual destructor. */ - virtual ~TBlob(); + virtual ~TBlob() { + deallocate(); + } /** * @brief Creates an new empty rvalue LockedMemory object. @@ -806,7 +808,8 @@ protected: } }; -#ifdef __clang__ +// These should not be exported for WIN32 to avoid usage of '_handle' and '_allocator' across CRT bounaries +#ifndef WIN32 extern template class INFERENCE_ENGINE_API_CLASS(InferenceEngine::TBlob); extern template class INFERENCE_ENGINE_API_CLASS(InferenceEngine::TBlob); extern template class INFERENCE_ENGINE_API_CLASS(InferenceEngine::TBlob); @@ -821,7 +824,7 @@ extern template class INFERENCE_ENGINE_API_CLASS(InferenceEngine::TBlob); extern template class INFERENCE_ENGINE_API_CLASS(InferenceEngine::TBlob); extern template class INFERENCE_ENGINE_API_CLASS(InferenceEngine::TBlob); -#endif // __clang__ +#endif /** * @brief Creates a blob with the given tensor descriptor. diff --git a/src/inference/src/ie_blob_common.cpp b/src/inference/src/ie_blob_common.cpp index e9be76b15b8..d168a581abe 100644 --- a/src/inference/src/ie_blob_common.cpp +++ b/src/inference/src/ie_blob_common.cpp @@ -85,24 +85,21 @@ Blob::Ptr make_shared_blob(const Blob::Ptr& inputBlob, Blob::~Blob() {} MemoryBlob::~MemoryBlob() {} -template -TBlob::~TBlob() { - free(); -} - -template class INFERENCE_ENGINE_API_CLASS(TBlob); -template class INFERENCE_ENGINE_API_CLASS(TBlob); -template class INFERENCE_ENGINE_API_CLASS(TBlob); -template class INFERENCE_ENGINE_API_CLASS(TBlob); -template class INFERENCE_ENGINE_API_CLASS(TBlob); -template class INFERENCE_ENGINE_API_CLASS(TBlob); -template class INFERENCE_ENGINE_API_CLASS(TBlob); -template class INFERENCE_ENGINE_API_CLASS(TBlob); -template class INFERENCE_ENGINE_API_CLASS(TBlob); -template class INFERENCE_ENGINE_API_CLASS(TBlob); -template class INFERENCE_ENGINE_API_CLASS(TBlob); -template class INFERENCE_ENGINE_API_CLASS(TBlob); -template class INFERENCE_ENGINE_API_CLASS(TBlob); -template class INFERENCE_ENGINE_API_CLASS(TBlob); +#ifndef WIN32 +template class TBlob; +template class TBlob; +template class TBlob; +template class TBlob; +template class TBlob; +template class TBlob; +template class TBlob; +template class TBlob; +template class TBlob; +template class TBlob; +template class TBlob; +template class TBlob; +template class TBlob; +template class TBlob; +#endif } // namespace InferenceEngine From 7a0d85a067ae50dcb52ec960383a9fc51af2e7b2 Mon Sep 17 00:00:00 2001 From: Mateusz Bencer Date: Wed, 30 Mar 2022 18:13:55 +0200 Subject: [PATCH 08/39] remove resize asserts (#11234) --- src/frontends/onnx/frontend/src/op/resize.cpp | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/src/frontends/onnx/frontend/src/op/resize.cpp b/src/frontends/onnx/frontend/src/op/resize.cpp index dd4a8ec3c94..a3352aa4604 100644 --- a/src/frontends/onnx/frontend/src/op/resize.cpp +++ b/src/frontends/onnx/frontend/src/op/resize.cpp @@ -142,7 +142,6 @@ OutputVector resize(const onnx_import::Node& node) { // in "tf_crop_and_resize" which is not handled now const auto inputs = node.get_ng_inputs(); const auto& data = inputs.at(0); - const auto& data_shape = data.get_partial_shape(); auto attrs = get_resize_attrs(node); @@ -150,12 +149,6 @@ OutputVector resize(const onnx_import::Node& node) { { attrs.shape_calculation_mode = default_opset::Interpolate::ShapeCalcMode::SIZES; const auto& sizes = inputs.at(3); - const auto& sizes_shape = sizes.get_partial_shape(); - - CHECK_VALID_NODE(node, - (sizes_shape.is_static() || data_shape.rank().is_static()), - " Data rank or shape of sizes input is required to be static."); - const auto scales = calculate_scales_based_on_sizes(data, sizes); return {std::make_shared(data, sizes, scales, attrs)}; @@ -164,12 +157,6 @@ OutputVector resize(const onnx_import::Node& node) { attrs.shape_calculation_mode = default_opset::Interpolate::ShapeCalcMode::SCALES; const auto& scales = inputs.at(2); - const auto& scales_shape = scales.get_partial_shape(); - - CHECK_VALID_NODE(node, - (scales_shape.is_static() || data_shape.rank().is_static()), - " Data rank or shape of scales input is required to be static."); - const auto output_shape = calculate_output_shape_based_on_scales(data, scales); return {std::make_shared(data, output_shape, scales, attrs)}; } @@ -181,9 +168,6 @@ OutputVector resize(const onnx_import::Node& node) { const auto& data = inputs.at(0); const auto& scales = inputs.at(1); - const auto& data_shape = data.get_partial_shape(); - const auto& scales_shape = scales.get_partial_shape(); - auto attrs = get_resize_attrs(node); if (attrs.mode == InterpolateMode::NEAREST) { @@ -193,10 +177,6 @@ OutputVector resize(const onnx_import::Node& node) { attrs.coordinate_transformation_mode = Transform_mode::ASYMMETRIC; } - CHECK_VALID_NODE(node, - (scales_shape.is_static() || data_shape.rank().is_static()), - " Data rank or shape of scales input is required to be static."); - const auto output_shape = calculate_output_shape_based_on_scales(data, scales); return {std::make_shared(data, output_shape, scales, attrs)}; } From c8720f122d502f21784db1a7cbe20ae7d3d52cfe Mon Sep 17 00:00:00 2001 From: Alexey Lebedev Date: Wed, 30 Mar 2022 19:43:07 +0300 Subject: [PATCH 09/39] [PYTHON API] lifetime test for CompiledModel and extension (#11120) * Add test for lifitime and extensions * Remove ExtendedModel --- .../python/src/openvino/runtime/ie_api.py | 14 ++--- src/bindings/python/tests/conftest.py | 58 ++++++++++++++++++ .../test_compiled_model.py | 11 +++- .../tests/test_inference_engine/test_core.py | 60 ++----------------- 4 files changed, 77 insertions(+), 66 deletions(-) diff --git a/src/bindings/python/src/openvino/runtime/ie_api.py b/src/bindings/python/src/openvino/runtime/ie_api.py index b8ec1b23866..a49f6f62ee5 100644 --- a/src/bindings/python/src/openvino/runtime/ie_api.py +++ b/src/bindings/python/src/openvino/runtime/ie_api.py @@ -334,23 +334,17 @@ class Core(CoreBase): ) -class ExtendedModel(CompiledModel): - """CompiledModel that additionally holds Core object.""" - - def __init__(self, core: Core, net: CompiledModel): - super().__init__(net) - self.core = core # needs to store Core object for CPU plugin - - def compile_model(model_path: str) -> CompiledModel: """Compact method to compile model with AUTO plugin. :param model_path: Path to file with model. :type model_path: str - :return: Extended version of `CompiledModel` that holds and keeps alive `Core` object. + :return: A compiled model. + :rtype: openvino.runtime.CompiledModel + """ core = Core() - return ExtendedModel(core, core.compile_model(model_path, "AUTO")) + return core.compile_model(model_path, "AUTO") class OVAny(OVAnyBase): diff --git a/src/bindings/python/tests/conftest.py b/src/bindings/python/tests/conftest.py index 3ba267affbd..8c527d9478d 100644 --- a/src/bindings/python/tests/conftest.py +++ b/src/bindings/python/tests/conftest.py @@ -8,6 +8,8 @@ import numpy as np import tests from pathlib import Path +from sys import platform +from openvino.runtime import Core def image_path(): @@ -29,6 +31,62 @@ def read_image(): return image +def get_model_with_template_extension(): + core = Core() + ir = bytes(b""" + + + + + + 1 + 3 + 22 + 22 + + + + + + + 1 + 3 + 22 + 22 + + + + + 1 + 3 + 22 + 22 + + + + + + + 1 + 3 + 22 + 22 + + + + + + + + +""") + if platform == "win32": + core.add_extension(library_path="openvino_template_extension.dll") + else: + core.add_extension(library_path="libopenvino_template_extension.so") + return core, core.read_model(ir) + + def model_path(is_myriad=False): path_to_repo = os.environ["MODELS_PATH"] if not is_myriad: diff --git a/src/bindings/python/tests/test_inference_engine/test_compiled_model.py b/src/bindings/python/tests/test_inference_engine/test_compiled_model.py index e0bb872bf4d..6b47b3dfc1a 100644 --- a/src/bindings/python/tests/test_inference_engine/test_compiled_model.py +++ b/src/bindings/python/tests/test_inference_engine/test_compiled_model.py @@ -5,7 +5,7 @@ import os import pytest import numpy as np -from ..conftest import model_path, read_image +from ..conftest import model_path, read_image, get_model_with_template_extension from openvino.runtime import Model, ConstOutput, Shape from openvino.runtime import Core, Tensor @@ -340,3 +340,12 @@ def test_direct_infer(device): assert np.argmax(res[comp_model.outputs[0]]) == 2 ref = comp_model.infer_new_request({"data": tensor}) assert np.array_equal(ref[comp_model.outputs[0]], res[comp_model.outputs[0]]) + + +def test_compiled_model_after_core_destroyed(device): + core, model = get_model_with_template_extension() + compiled = core.compile_model(model, device) + del core + del model + # check compiled and infer request can work properly after core object is destroyed + compiled([np.random.normal(size=list(input.shape)) for input in compiled.inputs]) diff --git a/src/bindings/python/tests/test_inference_engine/test_core.py b/src/bindings/python/tests/test_inference_engine/test_core.py index 6340e7c30c7..ac80e22e7b7 100644 --- a/src/bindings/python/tests/test_inference_engine/test_core.py +++ b/src/bindings/python/tests/test_inference_engine/test_core.py @@ -11,7 +11,8 @@ import openvino.runtime.opset8 as ov from openvino.runtime import Model, Core, CompiledModel, Tensor, PartialShape, Extension,\ tensor_from_file, compile_model -from ..conftest import model_path, model_onnx_path, plugins_path, read_image +from ..conftest import model_path, model_onnx_path, plugins_path, read_image, \ + get_model_with_template_extension test_net_xml, test_net_bin = model_path() @@ -254,60 +255,7 @@ def test_unregister_plugin(device): @pytest.mark.template_extension def test_add_extension_template_extension(device): - ir = bytes(b""" - - - - - - 1 - 3 - 22 - 22 - - - - - - - 1 - 3 - 22 - 22 - - - - - 1 - 3 - 22 - 22 - - - - - - - 1 - 3 - 22 - 22 - - - - - - - - -""") - - core = Core() - if platform == "win32": - core.add_extension(library_path="openvino_template_extension.dll") - else: - core.add_extension(library_path="libopenvino_template_extension.so") - model = core.read_model(model=ir) + core, model = get_model_with_template_extension() assert isinstance(model, Model) before_reshape = PartialShape([1, 3, 22, 22]) @@ -315,6 +263,8 @@ def test_add_extension_template_extension(device): new_shapes = {"in_data": after_reshape} assert model.input().partial_shape == before_reshape model.reshape(new_shapes) + # compile to check objects can be destroyed + # in order core -> model -> compiled compiled = core.compile_model(model, device) assert compiled.input().partial_shape == after_reshape From be6db5d69ab30dc16b9ca2cbdd53cdd21a783551 Mon Sep 17 00:00:00 2001 From: Oleg Pipikin Date: Wed, 30 Mar 2022 19:48:29 +0300 Subject: [PATCH 10/39] Fix for str_to_container if string value has whitespaces (#10224) * Fix for str_to_container if string value has whitespaces * Add test * Add trim for leading and trailing whitespaces * Apply comments * Apply comments 2 * Apply comments 3 --- src/frontends/ir/src/rt_info_deserializer.cpp | 2 +- src/frontends/ir/src/utils.cpp | 15 +++++++++++++++ src/frontends/ir/src/utils.hpp | 3 +++ .../ir_serialization/rt_info_deserialization.cpp | 4 ++-- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/frontends/ir/src/rt_info_deserializer.cpp b/src/frontends/ir/src/rt_info_deserializer.cpp index 5d8fff73eac..4da107033cc 100644 --- a/src/frontends/ir/src/rt_info_deserializer.cpp +++ b/src/frontends/ir/src/rt_info_deserializer.cpp @@ -17,7 +17,7 @@ void RTInfoDeserializer::on_adapter(const std::string& name, ValueAccessor return; if (auto a = as_type>>(&adapter)) { std::set ss; - str_to_container(val, ss); + str_to_set_of_strings(val, ss); a->set(ss); } else { IE_THROW() << "Not implemented"; diff --git a/src/frontends/ir/src/utils.cpp b/src/frontends/ir/src/utils.cpp index 15bf031443a..6207eb261c5 100644 --- a/src/frontends/ir/src/utils.cpp +++ b/src/frontends/ir/src/utils.cpp @@ -98,4 +98,19 @@ bool get_dimension_from_attribute(const pugi::xml_node& node, const std::string& return true; } +void str_to_set_of_strings(const std::string& value, std::set& res) { + std::stringstream ss(value); + std::string field; + while (getline(ss, field, ',')) { + // trim leading and trailing whitespaces + auto strBegin = field.find_first_not_of(" "); + if (strBegin == std::string::npos) + IE_THROW() << "Cannot get a set of strings from \"" << value << "\". Value \"" << field + << "\" is incorrect"; + auto strRange = field.find_last_not_of(" ") - strBegin + 1; + + res.insert(field.substr(strBegin, strRange)); + } +} + } // namespace ov diff --git a/src/frontends/ir/src/utils.hpp b/src/frontends/ir/src/utils.hpp index d3ed15ef306..f8f53da7ebd 100644 --- a/src/frontends/ir/src/utils.hpp +++ b/src/frontends/ir/src/utils.hpp @@ -32,6 +32,9 @@ void str_to_container(const std::string& value, T& res) { res.insert(res.end(), val); } } +// separated function for set to keep whitespaces in values +// because stringstream splits its values with whitespace delimiter +void str_to_set_of_strings(const std::string& value, std::set& res); template bool getParameters(const pugi::xml_node& node, const std::string& name, std::vector& value) { diff --git a/src/tests/functional/inference_engine/ir_serialization/rt_info_deserialization.cpp b/src/tests/functional/inference_engine/ir_serialization/rt_info_deserialization.cpp index 6902ee30b47..7b2bc95c24d 100644 --- a/src/tests/functional/inference_engine/ir_serialization/rt_info_deserialization.cpp +++ b/src/tests/functional/inference_engine/ir_serialization/rt_info_deserialization.cpp @@ -472,7 +472,7 @@ TEST_F(RTInfoDeserialization, NodeV11) { - + @@ -553,7 +553,7 @@ TEST_F(RTInfoDeserialization, NodeV11) { auto result = f->get_result(); check_old_api_map_order(result->get_rt_info(), std::vector({0, 3, 1, 2})); auto round = result->get_input_node_ptr(0); - check_fused_names(round->get_rt_info(), "Round1,Round2"); + check_fused_names(round->get_rt_info(), "Round 1,Round 2"); // read IR v11 with new API { From 4c7050f6a9eed6470de8d0d4c243129e3d6ff40b Mon Sep 17 00:00:00 2001 From: Anastasia Kuporosova Date: Wed, 30 Mar 2022 20:26:36 +0300 Subject: [PATCH 11/39] [Python API] Improve configuration files (#10960) * [Python API] Improve configuration files * fix config files * update setup.cfd + change quotes * move all codestyle checks to py_checks job * update requirements_test.txt * fix codestyle according to flake-docstring * fix * fix mypy * apply comments --- .ci/openvino-onnx/Jenkinsfile | 213 -------- .ci/openvino-onnx/watchdog/Jenkinsfile | 65 --- .ci/openvino-onnx/watchdog/requirements.txt | 6 - .ci/openvino-onnx/watchdog/src/git_wrapper.py | 108 ---- .../watchdog/src/jenkins_wrapper.py | 91 ---- .ci/openvino-onnx/watchdog/src/main.py | 89 --- .../watchdog/src/ms_teams_communicator.py | 128 ----- .ci/openvino-onnx/watchdog/src/watchdog.py | 505 ------------------ .github/workflows/py_checks.yml | 69 ++- .../classification_sample_async.py | 2 +- .../hello_query_device/hello_query_device.py | 2 +- .../model_creation_sample.py | 2 +- samples/python/setup.cfg | 6 +- samples/python/speech_sample/arg_parser.py | 4 +- samples/python/speech_sample/file_options.py | 12 +- samples/python/speech_sample/speech_sample.py | 2 +- samples/python/speech_sample/utils.py | 8 +- src/bindings/python/requirements_test.txt | 48 +- src/bindings/python/setup.cfg | 64 ++- .../src/compatibility/openvino/setup.cfg | 6 +- src/bindings/python/tox.ini | 75 --- src/bindings/python/wheel/setup.py | 414 +++++++------- 22 files changed, 378 insertions(+), 1541 deletions(-) delete mode 100644 .ci/openvino-onnx/Jenkinsfile delete mode 100644 .ci/openvino-onnx/watchdog/Jenkinsfile delete mode 100644 .ci/openvino-onnx/watchdog/requirements.txt delete mode 100644 .ci/openvino-onnx/watchdog/src/git_wrapper.py delete mode 100644 .ci/openvino-onnx/watchdog/src/jenkins_wrapper.py delete mode 100644 .ci/openvino-onnx/watchdog/src/main.py delete mode 100644 .ci/openvino-onnx/watchdog/src/ms_teams_communicator.py delete mode 100644 .ci/openvino-onnx/watchdog/src/watchdog.py delete mode 100644 src/bindings/python/tox.ini diff --git a/.ci/openvino-onnx/Jenkinsfile b/.ci/openvino-onnx/Jenkinsfile deleted file mode 100644 index 6b8b19193d4..00000000000 --- a/.ci/openvino-onnx/Jenkinsfile +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright (C) 2018-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -DOCKER_CONTAINER_NAME= "openvino-onnx-ci-container" -DOCKER_IMAGE_TAG = "openvino-onnx-ci-image" -ONNX_MODEL_ZOO_SHA = "d58213534f2a4d1c4b19ba62b3bb5f544353256e" - -BACKEND_CONFIGURATIONS = [ - [ name: "Release", build_type: "Release" ], - [ name: "Debug", build_type: "Debug" ], -] - -// workaround for aborting previous builds on PR update -@NonCPS - def stopPreviousRunningBuilds() { - def jobname = env.JOB_NAME - if (jobname.startsWith("onnx-ci/openvino onnx ci/openvino/PR")){ - def buildnum = env.BUILD_NUMBER.toInteger() - def job = Jenkins.instance.getItemByFullName(jobname) - def job_newest = job.builds.first() - for (build in job.builds.reverse()[0..<-1]) { - if (build.isBuilding()){ - echo "Stop task = ${build} because newest #${job_newest} is on the way" - build.doStop(); - continue; - } - } - } - - } - -def getGitPrInfo(String project, String workdir) { - def gitPrInfo = [ - prAuthorEmail : "", - commitAuthorEmail : "", - commitHash : "", - commitSubject : "" - ] - try { - dir ("${workdir}/${project}") { - gitPrInfo.prAuthorEmail = sh (script: 'git log -1 --pretty="format:%ae" ', returnStdout: true).trim() - gitPrInfo.commitAuthorEmail = sh (script: 'git log -1 --pretty="format:%ce" ', returnStdout: true).trim() - gitPrInfo.commitSubject = sh (script: 'git log -1 --pretty="format:%H" ', returnStdout: true).trim() - gitPrInfo.commitHash = sh (script: 'git log -1 --pretty="format:%s" ', returnStdout: true).trim() - } - } - catch(e) { - echo "Failed to retrieve ${project} git repository information!" - echo "ERROR: ${e}" - } - return gitPrInfo -} - -def notifyByEmail(def gitPrInfo) { - stage('Notify') { - String notifyPeople = "${gitPrInfo.prAuthorEmail}, ${gitPrInfo.commitAuthorEmail}" - emailext ( - subject: "OpenVino CI: PR ${CHANGE_ID} ${currentBuild.result}!", - body: """ - Status: ${currentBuild.result} - Pull Request Title: ${CHANGE_TITLE} - Pull Request: ${CHANGE_URL} - Branch: ${CHANGE_BRANCH} - Commit Hash: ${gitPrInfo.commitSubject} - Commit Subject: ${gitPrInfo.commitHash} - Jenkins Build: ${RUN_DISPLAY_URL} - """, - to: "${notifyPeople}" - ) - } -} - -def gitSubmoduleUpdate(String repository_name, String workdir) { - dir ("${workdir}/${repository_name}") { - sh label: "Init ${repository_name} submodules", - script: - """ - git submodule init && git submodule update \ - --init \ - --no-fetch \ - --recursive - """ - } -} - -def prepare_repository(String workdir) { - dir("${workdir}") { - println "Preparing repository in directory: ${workdir}" - checkout scm - gitSubmoduleUpdate(PROJECT_NAME, workdir) - } -} - -def updateModels() { - sh """ - ./src/bindings/python/tests/test_onnx/model_zoo_preprocess.sh -d ${HOME}/ONNX_CI/models_data -o -s ${ONNX_MODEL_ZOO_SHA} - """ -} - -def get_docker_container_name(Map configuration){ - println "RUN get_docker_container_name for ${configuration.name}" - String docker_container_name = "${DOCKER_CONTAINER_NAME}_${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}" - return docker_container_name -} - -def buildDockerImage(Map configuration, String workdir) { - String docker_image_tag = "${DOCKER_IMAGE_TAG}_${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}".toLowerCase() - println "docker_image_tag: ${docker_image_tag}" - updateModels() - sh """ - docker build --tag=${docker_image_tag} \ - --build-arg BUILD_TYPE=${configuration.build_type} \ - --file=.ci/openvino-onnx/Dockerfile \ - --build-arg http_proxy=${HTTP_PROXY} \ - --build-arg https_proxy=${HTTPS_PROXY} . - """ -} - -def runTests(Map configuration, String workdir) { - println "Run tests for ${configuration.name}" - String docker_image_tag = "${DOCKER_IMAGE_TAG}_${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}".toLowerCase() - - String docker_container_name = get_docker_container_name(configuration) - - // Run only basic unit tests in Debug configuration - if (configuration.build_type == "Debug") { - sh """ - docker run --name ${docker_container_name} ${docker_image_tag} - """ - } - - // Run unit-tests AND large model tests by default - else { - sh """ - docker run --name ${docker_container_name} \ - --volume ${HOME}/ONNX_CI/models_data/model_zoo/onnx_model_zoo_${ONNX_MODEL_ZOO_SHA}:/root/.onnx/model_zoo/onnx_model_zoo \ - --volume ${HOME}/ONNX_CI/data/model_zoo/MSFT:/root/.onnx/model_zoo/MSFT \ - ${docker_image_tag} /bin/bash -c "tox && tox -e zoo_models" - """ - } -} - -def getConfigurationsMap() { - def configurationsMap = [:] - for (backend in BACKEND_CONFIGURATIONS) { - def configuration = backend.clone() - configurationsMap[configuration.name] = { - stage(configuration.name) { CONFIGURATION_WORKFLOW(configuration) } - } - } - return configurationsMap -} - -CONFIGURATION_WORKFLOW = { configuration -> - node("OpenVINO") { - String workdir = "${HOME}/workspace/${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}" - try { - PROJECT_NAME = "openvino" - stage("Clone repository") { - prepare_repository(workdir) - } - stage("Prepare Docker environment") { - dir("${workdir}") { - buildDockerImage(configuration, workdir) - } - } - stage("Run tests") { - timeout(time: 60, unit: 'MINUTES') { - runTests(configuration, workdir) - } - } - } - catch(e) { - // Set result to ABORTED if exception contains exit code of a process interrupted by SIGTERM - if ("$e".contains("143")) { - currentBuild.result = "ABORTED" - } else { - currentBuild.result = "FAILURE" - } - def gitPrInfo = getGitPrInfo(PROJECT_NAME, workdir) - notifyByEmail(gitPrInfo) - } - finally { - stage("Cleanup") { - String docker_container_name = get_docker_container_name(configuration) - sh """ - docker rm -f ${docker_container_name} - rm -rf ${workdir} - """ - } - } - } -} - -pipeline { - agent none - options { - skipDefaultCheckout true - timeout(activity: true, time: 120, unit: 'MINUTES') - } - stages { - stage('Parallel CI') { - steps { - stopPreviousRunningBuilds() - script { - parallelStagesMap = getConfigurationsMap() - parallel parallelStagesMap - } - } - - } - } -} diff --git a/.ci/openvino-onnx/watchdog/Jenkinsfile b/.ci/openvino-onnx/watchdog/Jenkinsfile deleted file mode 100644 index b3edb3dcc75..00000000000 --- a/.ci/openvino-onnx/watchdog/Jenkinsfile +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (C) 2018-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -timeout(30) -{ - node(LABEL) { - - BUILD_WORKSPACE = "$WORKSPACE/$BUILD_NUMBER" - WATCHDOG_ROOT = "$BUILD_WORKSPACE/.ci/openvino-onnx/watchdog" - VENV_PATH = "${BUILD_WORKSPACE}/.wdvenv" - - try { - stage("Clone repository") { - dir ("$BUILD_WORKSPACE") { - checkout([$class: 'GitSCM', branches: [[name: "*/$BRANCH"]], - doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [], - userRemoteConfigs: [[credentialsId: "${GITHUB_KEY}", url: "${OPEN_VINO_URL}"]]]) - } - } - stage("Prepare environment") { - sh """#!/bin/bash - if [ ! -d ${VENV_PATH} ]; then - python3 -m venv ${VENV_PATH} - source ${VENV_PATH}/bin/activate - pip install -r ${WATCHDOG_ROOT}/requirements.txt - fi - """ - } - stage("Run script") { - withCredentials([ - usernamePassword(credentialsId: '7157091e-bc04-42f0-99fd-dc4da2922a55', - usernameVariable: 'username', - passwordVariable: 'password')]) - { - dir ("$BUILD_WORKSPACE") { - sh """#!/bin/bash - source ${VENV_PATH}/bin/activate - export PYTHONHTTPSVERIFY=0 - python ${WATCHDOG_ROOT}/src/main.py \ - --msteams-url=${MSTEAMS_URL_FILE} \ - --github-credentials '${username}' '${password}' \ - --github-org=${GITHUB_ORG} \ - --github-project=${GITHUB_PROJECT} \ - --jenkins-token=${JENKINS_TOKEN_FILE} \ - --jenkins-server=${JENKINS_SERVER} \ - --jenkins-user=${JENKINS_USER} \ - --ci-job=${CI_JOB_NAME} \ - --watchdog-job=${WATCHDOG_JOB_NAME} - """ - } - } - } - } catch (e) { - echo "$e" - currentBuild.result = "FAILURE" - } finally { - stage("Cleanup") { - sh """ - cd $BUILD_WORKSPACE - rm -rf ..?* .[!.]* * - """ - } - } - } -} diff --git a/.ci/openvino-onnx/watchdog/requirements.txt b/.ci/openvino-onnx/watchdog/requirements.txt deleted file mode 100644 index bab996b6438..00000000000 --- a/.ci/openvino-onnx/watchdog/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -python-jenkins==1.7.0 -retrying==1.3.3 -pygithub==1.51 -timeout-decorator==0.4.1 -requests==2.23.0 -wheel diff --git a/.ci/openvino-onnx/watchdog/src/git_wrapper.py b/.ci/openvino-onnx/watchdog/src/git_wrapper.py deleted file mode 100644 index 4b42ab7c8bb..00000000000 --- a/.ci/openvino-onnx/watchdog/src/git_wrapper.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/python3 - -# Copyright (C) 2018-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import logging -import timeout_decorator -from datetime import datetime -from retrying import retry -from github import Github, GithubException - -# Logging -logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s') -log = logging.getLogger(__name__) -log.setLevel(logging.INFO) - -_RETRY_LIMIT = 3 -_RETRY_COOLDOWN_MS = 2000 -_REQUEST_TIMEOUT_S = 10 - - -class GitWrapper: - """Class wrapping PyGithub API. - - The purpose of this class is to wrap methods from PyGithub API used in Watchdog, for less error-prone and - more convenient use. Docs for used API, including wrapped methods can be found at: - https://pygithub.readthedocs.io/en/latest/introduction.html - - :param github_credentials: Credentials used for GitHub - :param repository: GitHub repository name - :param project: GitHub project name - :type github_credentials: String - :type repository: String - :type project: String - """ - - def __init__(self, github_credentials, repository, project): - self.git = Github(*github_credentials) - self.repository = repository - self.project = project - self.github_credentials = github_credentials - - @retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS) - def get_git_time(self): - """Retrieve time from GitHub. - - Used to reliably determine time during Watchdog run. - - :return: Datetime object describing current time - :rtype: datetime - """ - try: - datetime_object = self._get_git_time() - except ValueError as e: - raise GitWrapperError(str(e)) - except GithubException as e: - message = 'GitHub Exception during API status retrieval. Exception: {}'.format(str(e)) - raise GitWrapperError(message) - except timeout_decorator.TimeoutError: - message = 'GitHub Exception during API status retrieval. Timeout during API request.' - raise GitWrapperError(message) - return datetime_object - - @retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS) - def get_pull_requests(self): - """Retrieve paginated list of pull requests from GitHub. - - :return: Paginated list of Pull Requests in GitHub repo - :rtype: github.PaginatedList.PaginatedList of github.PullRequest.PullRequest - """ - try: - prs = self._get_pull_requests() - except GithubException as e: - message = 'GitHub Exception during API status retrieval. Exception: {}'.format(str(e)) - raise GitWrapperError(message) - return prs - - @timeout_decorator.timeout(_REQUEST_TIMEOUT_S) - def _get_git_time(self): - """Private method retrieving time from GitHub. - - :return: Datetime object describing current time - :rtype: datetime - """ - datetime_string = self.git.get_api_status().raw_headers.get('date', '') - datetime_format = '%a, %d %b %Y %H:%M:%S %Z' - datetime_object = datetime.strptime(datetime_string, datetime_format) - return datetime_object - - @timeout_decorator.timeout(_REQUEST_TIMEOUT_S) - def _get_pull_requests(self): - """Private method retrieving pull requests from GitHub. - - :return: Paginated list of Pull Requests in GitHub repo - :rtype: github.PaginatedList.PaginatedList of github.PullRequest.PullRequest - """ - return self.git.get_organization(self.repository).get_repo(self.project).get_pulls() - - -class GitWrapperError(Exception): - """Base class for exceptions raised in GitWrapper. - - :param message Explanation of the error - """ - - def __init__(self, message): - self.message = message - log.exception(message) diff --git a/.ci/openvino-onnx/watchdog/src/jenkins_wrapper.py b/.ci/openvino-onnx/watchdog/src/jenkins_wrapper.py deleted file mode 100644 index 4abc345a08f..00000000000 --- a/.ci/openvino-onnx/watchdog/src/jenkins_wrapper.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/python3 - -# Copyright (C) 2018-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import requests -import jenkins -import logging -from retrying import retry - -# Logging -logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s') -log = logging.getLogger(__name__) -log.setLevel(logging.INFO) - -_RETRY_LIMIT = 3 -_RETRY_COOLDOWN_MS = 5000 - - -class JenkinsWrapper: - """Class wrapping Python-Jenkins API. - - The purpose of this class is to wrap methods from Python-Jenkins API used in Watchdog, for less error-prone and - more convenient use. Docs for used API, including wrapped methods can be found at: - https://python-jenkins.readthedocs.io/en/latest/ - - :param jenkins_token: Token used for Jenkins - :param jenkins_user: Username used to connect to Jenkins - :param jenkins_server: Jenkins server address - :type jenkins_token: String - :type jenkins_user: String - :type jenkins_server: String - """ - - def __init__(self, jenkins_token, jenkins_user, jenkins_server): - self.jenkins_server = jenkins_server - self.jenkins = jenkins.Jenkins(jenkins_server, username=jenkins_user, - password=jenkins_token) - - @retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS) - def get_build_console_output(self, job_name, build_number): - return self.jenkins.get_build_console_output(job_name, build_number) - - @retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS) - def get_job_info(self, job_name): - return self.jenkins.get_job_info(job_name) - - @retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS) - def get_build_info(self, job_name, build_number): - return self.jenkins.get_build_info(job_name, build_number) - - @retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS) - def get_queue_item(self, queue_id): - """Attempt to retrieve Jenkins job queue item. - - Exception communicating queue doesn't exist is expected, - in that case method returns empty dict. - - :param queue_id: Jenkins job queue ID number - :type queue_id: int - :return: Dictionary representing Jenkins job queue item - :rtype: dict - """ - try: - return self.jenkins.get_queue_item(queue_id) - except Exception as e: - # Exception 'queue does not exist' is expected behaviour when job is running - if 'queue' in str(e) and 'does not exist' in str(e): - return {} - else: - raise - - @retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS) - def get_idle_ci_hosts(self): - """Query Jenkins for idle servers. - - Send GET request to Jenkins server, querying for idle servers labeled - for OpenVino-ONNX CI job. - - :return: Number of idle hosts delegated to OpenVino-ONNX CI - :rtype: int - """ - jenkins_request_url = self.jenkins_server + 'label/ci&&onnx/api/json?pretty=true' - try: - log.info('Sending request to Jenkins: %s', jenkins_request_url) - r = requests.Request(method='GET', url=jenkins_request_url, verify=False) - response = self.jenkins.jenkins_request(r).json() - return int(response['totalExecutors']) - int(response['busyExecutors']) - except Exception as e: - log.exception('Failed to send request to Jenkins!\nException message: %s', str(e)) - raise diff --git a/.ci/openvino-onnx/watchdog/src/main.py b/.ci/openvino-onnx/watchdog/src/main.py deleted file mode 100644 index a0b7ae23bdf..00000000000 --- a/.ci/openvino-onnx/watchdog/src/main.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/python3 - -# Copyright (C) 2018-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import argparse -import sys -from watchdog import Watchdog - -DEFAULT_MSTEAMS_URL_FILE = '/home/lab_nerval/tokens/msteams_url' -DEFAULT_GITHUB_ORGANIZATION = 'openvinotoolkit' -DEFAULT_GITHUB_PROJECT = 'openvino' -DEFAULT_JENKINS_TOKEN_FILE = '/home/lab_nerval/tokens/crackerjack' -DEFAULT_JENKINS_SERVER = 'https://crackerjack.intel.com/' -DEFAULT_JENKINS_USER = 'lab_nerval' -DEFAULT_CI_JOB_NAME = 'onnx/OpenVino_CI' -DEFAULT_WATCHDOG_JOB_NAME = 'onnx/ci_watchdog' - - -def main(args): - """ - Read args passed to script, load tokens and run watchdog. - - Keyword arguments: - :param args: arguments parsed by argparse ArgumentParser - - :return: returns status code 0 on successful completion - - """ - jenkins_server = args.jenkins_server.strip() - jenkins_user = args.jenkins_user.strip() - jenkins_token = open(args.jenkins_token).read().replace('\n', '').strip() - msteams_url = open(args.msteams_url).read().replace('\n', '').strip() - github_credentials = args.github_credentials - github_org = args.github_org - github_project = args.github_project - ci_job = args.ci_job.strip() - watchdog_job = args.watchdog_job.strip() - quiet = args.quiet - - wd = Watchdog(jenkins_token=jenkins_token, - jenkins_server=jenkins_server, - jenkins_user=jenkins_user, - github_credentials=github_credentials, - git_org=github_org, - git_project=github_project, - msteams_url=msteams_url, - ci_job_name=ci_job, - watchdog_job_name=watchdog_job) - wd.run(quiet=quiet) - - return 0 - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument('--msteams-url', help='Path to MS Teams channel url to communicate messages.', - default=DEFAULT_MSTEAMS_URL_FILE, action='store', required=False) - - parser.add_argument('--github-credentials', help='GitHub user credentials to access repo.', - nargs="+", required=True) - - parser.add_argument('--github-org', help='Name of organization on GitHub.', - default=DEFAULT_GITHUB_ORGANIZATION, action='store', required=False) - - parser.add_argument('--github-project', help='Name of project on GitHub.', - default=DEFAULT_GITHUB_PROJECT, action='store', required=False) - - parser.add_argument('--jenkins-token', help='Path to Jenkins user token to access build info.', - default=DEFAULT_JENKINS_TOKEN_FILE, action='store', required=False) - - parser.add_argument('--jenkins-server', help='Jenkins server address.', - default=DEFAULT_JENKINS_SERVER, action='store', required=False) - - parser.add_argument('--jenkins-user', help='Jenkins user used to log in.', - default=DEFAULT_JENKINS_USER, action='store', required=False) - - parser.add_argument('--ci-job', help='Jenkins CI job name.', - default=DEFAULT_CI_JOB_NAME, action='store', required=False) - - parser.add_argument('--watchdog-job', help='Jenkins CI Watchdog job name.', - default=DEFAULT_WATCHDOG_JOB_NAME, action='store', required=False) - - parser.add_argument('--quiet', help="Quiet mode - doesn\'t send message to communicator.", - action='store_true', required=False) - - args = parser.parse_args() - sys.exit(main(args)) diff --git a/.ci/openvino-onnx/watchdog/src/ms_teams_communicator.py b/.ci/openvino-onnx/watchdog/src/ms_teams_communicator.py deleted file mode 100644 index 47765f35664..00000000000 --- a/.ci/openvino-onnx/watchdog/src/ms_teams_communicator.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/python3 - -# Copyright (C) 2018-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import requests - - -class MSTeamsCommunicator: - """Class communicating with MSTeams using Incoming Webhook. - - The purpose of this class is to use MSTeams API to send message. - Docs for used API, including wrapped methods can be found at: - https://docs.microsoft.com/en-us/outlook/actionable-messages/send-via-connectors - """ - - def __init__(self, _ci_alerts_channel_url): - self._ci_alerts_channel_url = _ci_alerts_channel_url - self._queued_messages = { - self._ci_alerts_channel_url: [], - } - - @property - def messages(self): - """ - Get list of queued messages. - - :return: List of queued messages - :return type: List[String] - """ - return self._queued_messages.values() - - def queue_message(self, message): - """ - Queue message to be sent later. - - :param message: Message content - :type message: String - """ - self._queued_messages[self._ci_alerts_channel_url].append(message) - - def _parse_text(self, watchdog_log, message): - """ - Parse text to display as alert. - - :param watchdog_log: Watchdog log content - :param message: Unparsed message content - :type watchdog_log: String - :type message: String - """ - message_split = message.split('\n') - log_url = None - if len(message_split) == 3: - log_url = message_split[-1] - title = message_split[0] - text = message_split[1] - header = watchdog_log.split(' - ') - header_formatted = '{} - [Watchdog Log]({})'.format(header[0], header[1]) - return title, log_url, '{}\n\n{}'.format(header_formatted, text) - - def _json_request_content(self, title, log_url, text_formatted): - """ - Create final json request to send message to MS Teams channel. - - :param title: Title of alert - :param log_url: URL to PR - :param text_formatted: General content of alert - finally formatted - :type title: String - :type title: String - :type title: String - """ - data = { - '@context': 'https://schema.org/extensions', - '@type': 'MessageCard', - 'themeColor': '0072C6', - 'title': title, - 'text': text_formatted, - 'potentialAction': - [ - { - '@type': 'OpenUri', - 'name': 'Open PR', - 'targets': - [ - { - 'os': 'default', - 'uri': log_url, - }, - ], - }, - ], - } - return data - - def _send_to_channel(self, watchdog_log, message_queue, channel_url): - """ - Send MSTeams message to specified channel. - - :param watchdog_log: Watchdog log content - :param message_queue: Queued messages to send - :param channel_url: Channel url - :type watchdog_log: String - :type message_queue: String - :type channel_url: String - - """ - for message in message_queue: - title, log_url, text_formatted = self._parse_text(watchdog_log, message) - data = self._json_request_content(title, log_url, text_formatted) - - try: - requests.post(url=channel_url, json=data) - except Exception as ex: - raise Exception('!!CRITICAL!! MSTeamsCommunicator: Could not send message ' - 'due to {}'.format(ex)) - - def send_message(self, watchdog_log, quiet=False): - """ - Send queued messages as single communication. - - :param watchdog_log: Watchdog log content - :param quiet: Flag for disabling sending report through MS Teams - :type watchdog_log: String - :type quiet: Boolean - """ - for channel, message_queue in self._queued_messages.items(): - if not quiet and message_queue: - self._send_to_channel(watchdog_log, message_queue, channel) diff --git a/.ci/openvino-onnx/watchdog/src/watchdog.py b/.ci/openvino-onnx/watchdog/src/watchdog.py deleted file mode 100644 index 3edbe478683..00000000000 --- a/.ci/openvino-onnx/watchdog/src/watchdog.py +++ /dev/null @@ -1,505 +0,0 @@ -#!/usr/bin/python3 - -# Copyright (C) 2018-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import datetime -import time -import re -import logging -import requests -from ms_teams_communicator import MSTeamsCommunicator -from jenkins_wrapper import JenkinsWrapper -from jenkins import NotFoundException -from git_wrapper import GitWrapper, GitWrapperError -import os -import json - -# Logging -logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s') -log = logging.getLogger(__name__) -log.setLevel(logging.INFO) - -# Watchdog static constant variables -_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -_BUILD_DURATION_THRESHOLD = datetime.timedelta(minutes=60) -_CI_START_THRESHOLD = datetime.timedelta(minutes=30) -_AWAITING_JENKINS_THRESHOLD = datetime.timedelta(minutes=5) -_WATCHDOG_DIR = os.path.expanduser('~') -_PR_REPORTS_CONFIG_KEY = 'pr_reports' -_CI_BUILD_FAIL_MESSAGE = 'ERROR: py3: commands failed' -_CI_BUILD_SUCCESS_MESSAGE = 'py3: commands succeeded' -_GITHUB_CI_CHECK_NAME = 'OpenVINO-ONNX' - -INTERNAL_ERROR_MESSAGE_HEADER = '!!! --- !!! INTERNAL WATCHDOG ERROR !!! --- !!!' -ERROR_MESSAGE_HEADER = '!!! OpenVino-ONNX CI Error !!!' -WARNING_MESSAGE_HEADER = 'OpenVino-ONNX CI WARNING' -INFO_MESSAGE_HEADER = 'OpenVino-ONNX CI INFO' - - -class Watchdog: - """Class describing OpenVino-ONNX-CI Watchdog. - - Watchdog connects to GitHub and retrieves the list of current pull requests (PRs) in - OpenVino repository. Then it connects to specified Jenkins server to - check CI jobs associated with every PR. Watchdog verifies time durations for Jenkins - initial response, job queue and execution against time treshold constants. Every fail - is logged and reported through MS Teams communicators. - - :param jenkins_token: Token used for Jenkins - :param jenkins_server: Jenkins server address - :param jenkins_user: Username used to connect to Jenkins - :param github_credentials: Credentials used to connect to GitHub - :param msteams_url: URL used to connect to MS Teams channel - :param ci_job_name: OpenVino-ONNX CI job name used in Jenkins - :param watchdog_job_name: Watchdog job name used in Jenkins - :type jenkins_token: String - :type jenkins_server: String - :type jenkins_user: String - :type github_credentials: String - :type msteams_url: String - :type ci_job_name: String - :type watchdog_job_name: String - - .. note:: - Watchdog and OpenVino-ONNX CI job must be placed on the same Jenkins server. - """ - - def __init__(self, jenkins_token, jenkins_server, jenkins_user, github_credentials, git_org, - git_project, msteams_url, ci_job_name, watchdog_job_name): - self._config_path = os.path.join(_WATCHDOG_DIR, '{}/.{}_ci_watchdog.json'.format(_WATCHDOG_DIR, git_project)) - # Jenkins Wrapper object for CI job - self._jenkins = JenkinsWrapper(jenkins_token, - jenkins_user=jenkins_user, - jenkins_server=jenkins_server) - # Load GitHub token and log in, retrieve pull requests - self._git = GitWrapper(github_credentials, repository=git_org, project=git_project) - # Create MS Teams api object - self._msteams_hook = MSTeamsCommunicator(msteams_url) - self._ci_job_name = ci_job_name.lower() - self._watchdog_job_name = watchdog_job_name - # Read config file - self._config = self._read_config_file() - # Time at Watchdog initiation - self._now_time = datetime.datetime.now() - self._current_prs = {} - self._ms_teams_enabled = True - - def run(self, quiet=False): - """Run main watchdog logic. - - Retrieve list of pull requests and pass it to the method responsible for checking them. - - :param quiet: Flag for disabling sending report through communicator - :type quiet: Boolean - """ - try: - pull_requests = self._git.get_pull_requests() - except GitWrapperError: - message = 'Failed to retrieve Pull Requests!' - log.exception(message) - self._queue_message(message, message_severity='internal') - # Check all pull requests - for pr in pull_requests: - try: - self._check_pr(pr) - except Exception as e: - log.exception(str(e)) - self._queue_message(str(e), message_severity='internal', pr=pr) - self._update_config() - self._send_message(quiet=quiet) - - def _read_config_file(self): - """Read Watchdog config file stored on the system. - - The file stores every fail already reported along with timestamp. This - mechanism is used to prevent Watchdog from reporting same failure - multiple times. In case there's no config under the expected path, - appropriate data structure is created and returned. - - :return: Returns dict of dicts with reported fails with their timestamps - :rtype: dict of dicts - """ - if os.path.isfile(self._config_path): - log.info('Reading config file in: {}'.format(self._config_path)) - file = open(self._config_path, 'r') - data = json.load(file) - else: - log.info('No config file found in: {}'.format(self._config_path)) - data = {_PR_REPORTS_CONFIG_KEY: {}} - return data - - def _check_pr(self, pr): - """Check pull request (if there's no reason to skip). - - Retrieve list of statuses for every PR's last commit and interpret them. Filters out statuses - unrelated to OpenVino-ONNX Jenkins CI and passes relevant statuses to method that interprets them. - If no commit statuses related to Jenkins are available after time defined by - **_AWAITING_JENKINS_THRESHOLD** calls appropriate method to check for builds waiting in queue. - - :param pr: GitHub Pull Requests - :type pr: github.PullRequest.PullRequest - """ - log.info('===============================================') - log.info('Checking PR#{}'.format(pr.number)) - # Get last Jenkins status - last_status = self._get_last_status(pr) - # Append PR checked in current run for Watchdog config - self._current_prs[str(pr.number)] = self._get_pr_timestamps(pr, last_status) - if self._should_ignore(pr) or self._updated_since_last_run(pr): - log.info('Ignoring PR#{}'.format(pr.number)) - - return - - # Calculate time passed since PR update (any commit, merge or comment) - pr_time_delta = self._now_time - pr.updated_at - if last_status: - # Interpret found CI statuses - log.info('Last status: {} at {}'.format(last_status.description, last_status.updated_at)) - self._interpret_status(last_status, pr) - elif pr_time_delta > _CI_START_THRESHOLD: - # If there's no status after assumed time - check if build is waiting in queue - log.info('CI for PR {}: NO JENKINS STATUS YET'.format(pr.number)) - self._check_missing_status(pr) - - @staticmethod - def _get_pr_timestamps(pr, last_status): - """Get dict containing PR timestamp and last status timestamp. - - :param pr: Single PR being currently checked - :type pr: github.PullRequest.PullRequest - - :return: Dictionary with PR and last status update timestamps - :rtype: dict - """ - pr_timestamp = time.mktime(pr.updated_at.timetuple()) - if last_status: - status_timestamp = time.mktime(last_status.updated_at.timetuple()) - else: - status_timestamp = None - pr_dict = {'pr_timestamp': pr_timestamp, - 'status_timestamp': status_timestamp} - return pr_dict - - @staticmethod - def _get_last_status(pr): - """Get last commit status posted from Jenkins. - - :param pr: Single PR being currently checked - :type pr: github.PullRequest.PullRequest - - :return: Either last PR status posted from Jenkins or None - :rtype: github.CommitStatus.CommitStatus - """ - # Find last commit in PR - last_commit = pr.get_commits().reversed[0] - # Get statuses and filter them to contain only those related to Jenkins CI - # and check if CI in Jenkins started - statuses = last_commit.get_statuses() - jenk_statuses = [stat for stat in statuses if - _GITHUB_CI_CHECK_NAME in stat.context] - try: - last_status = jenk_statuses[0] - except IndexError: - last_status = None - return last_status - - @staticmethod - def _should_ignore(pr): - """Determine if PR should be ignored. - - :param pr: Single PR being currently checked - :type pr: github.PullRequest.PullRequest - - :return: Returns True if PR should be ignored - :rtype: Bool - """ - # Ignore PR if it has WIP label or WIP in title - if 'WIP' in pr.title: - log.info('PR#{} should be ignored. WIP tag in title.'.format(pr.number)) - return True - - label_names = [label.name for label in pr.labels] - if 'WIP' in label_names: - log.info('PR#{} should be ignored. WIP label present.'.format(pr.number)) - return True - - # Ignore PR if base ref is not master - if 'master' not in pr.base.ref: - log.info('PR#{} should be ignored. Base ref is not master'.format(pr.number)) - return True - - # Ignore PR if mergeable state is 'dirty' or 'behind'. - # Practically this ignores PR in case of merge conflicts - ignored_mergeable_states = ['behind', 'dirty', 'draft'] - if pr.mergeable_state in ignored_mergeable_states: - log.info('PR#{} should be ignored. Mergeable state is {}. '.format(pr.number, pr.mergeable_state)) - return True - - # If no criteria for ignoring PR are met - return false - return False - - def _updated_since_last_run(self, pr): - # Ignore if PR was already checked and there was no update in meantime - pr_number = str(pr.number) - current_pr_timestamps = self._current_prs.get(pr_number) - last_pr_timestamps = self._config[_PR_REPORTS_CONFIG_KEY].get(pr_number) - if current_pr_timestamps == last_pr_timestamps: - log.info('PR#{} - No update since last check'.format(pr.number)) - return True - else: - return False - - def _check_missing_status(self, pr): - """Verify if missing status is expected. - - This method checks if CI build for last was scheduled and still waits in queue for - executor. - - :param pr: Single PR being currently checked - :type pr: github.PullRequest.PullRequest - """ - pr_time_delta = self._now_time - pr.updated_at - try: - build_number = self._build_scheduled(pr) - if self._build_in_queue(pr, build_number): - message = ('PR# {}: build waiting in queue after {} minutes.' - .format(pr.number, pr_time_delta.seconds / 60)) - severity = 'warning' - else: - message = ('PR# {}: missing status on GitHub after {} minutes.' - .format(pr.number, pr_time_delta.seconds / 60)) - severity = 'error' - self._queue_message(message, message_severity=severity, pr=pr) - except TypeError: - log.info('Committer outside of OpenVino organization') - - def _build_scheduled(self, pr): - """Check if Jenkins build corresponding to PR was scheduled. - - This method takes last Jenkins build for given PR and compares hash from Jenkins console output - and sha from PR object to determine if CI build for appropriate commit was scheduled. - - :param pr: Single PR being currently checked - :type pr: github.PullRequest.PullRequest - - :return: Returns build number or -1 if no build found - :rtype: int - """ - pr_number = str(pr.number) - project_name_full = self._ci_job_name + '/PR-' + pr_number - - try: - # Retrieve console output from last Jenkins build for job corresponding to this PR - last_build_number = self._jenkins.get_job_info(project_name_full)['lastBuild']['number'] - console_output = self._jenkins.get_build_console_output(project_name_full, last_build_number) - # Check if CI build was scheduled - commit hash on GH must match hash in last Jenkins build console output - # Retrieve hash from Jenkins output - match_string = '(?:Obtained .ci/[a-zA-Z/]+Jenkinsfile from ([a-z0-9]{40}))' - retrieved_sha = re.search(match_string, console_output).group(1) - if retrieved_sha == pr.get_commits().reversed[0].sha: - return last_build_number - else: - return -1 - except (NotFoundException, AttributeError, requests.exceptions.HTTPError): - message = ('PR #{}: Jenkins build corresponding to commit {} not found!' - .format(pr_number, pr.get_commits().reversed[0].sha)) - self._queue_message(message, message_severity='error', pr=pr) - return -1 - - def _build_in_queue(self, pr, build_number): - """Check if Jenkins build waits in queue. - - This method verifies if CI build is waiting in queue based on console output. - - :param pr: Single PR being currently checked - :param build_number: Jenkins build number to retrieve console output from - :type pr: github.PullRequest.PullRequest - :type build_number: int - - :return: Returns True if CI build is waiting in queue - :rtype: Bool - """ - pr_number = str(pr.number) - project_name_full = self._ci_job_name + '/PR-' + pr_number - # Retrieve console output - try: - console_output = self._jenkins.get_build_console_output(project_name_full, build_number) - except NotFoundException: - return False - # Check if build is waiting in queue (and not already running on an executor) - if 'Waiting for next available executor on' in console_output \ - and 'Running on' not in console_output: - log.info('CI for PR %s: WAITING IN QUEUE', pr_number) - return True - else: - return False - - def _interpret_status(self, status, pr): - """ - Verify GitHub status passed to the method. - - This method verifies last commit status for given PR, calling appropriate methods - to further validate the status. - - :param status: GitHub commit status - :param pr: Single PR being currently checked - :type status: github.CommitStatus.CommitStatus - :type pr: github.PullRequest.PullRequest - """ - try: - # Retrieve build number for Jenkins build related to this PR - build_number = self._retrieve_build_number(status.target_url) - # CI build finished - verify if expected output is present - finished_statuses = ['Build finished', 'This commit cannot be built', 'This commit looks good'] - pending_statuses = ['This commit is being built', 'Testing in progress', - 'This commit is scheduled to be built'] - if any(phrase in status.description for phrase in finished_statuses): - self._check_finished(pr, build_number) - # CI build in progress - verify timeouts for build queue and duration - elif any(phrase in status.description for phrase in pending_statuses): - self._check_in_progress(pr, build_number) - else: - message = 'ONNX CI job for PR# {}: unrecognized status: {}'.format(pr.number, status.description) - self._queue_message(message, message_severity='error', pr=pr) - except Exception: - # Log Watchdog internal error in case any status can't be properly verified - message = 'Failed to verify status "{}" for PR# {}'.format(status.description, pr.number) - log.exception(message) - self._queue_message(message, message_severity='internal', pr=pr) - - def _retrieve_build_number(self, url): - """Retrieve Jenkins CI job build number from URL address coming from GitHub commit status. - - :param url: URL address from GitHub commit status - :type url: String - - :return: Returns build number - :rtype: int - """ - # Retrieve the build number from url string - match_obj = re.search('(?:/PR-[0-9]+/)([0-9]+)', url) - try: - number = int(match_obj.group(1)) - return number - except Exception: - log.exception('Failed to retrieve build number from url link: %s', url) - raise - - def _queue_message(self, message, message_severity='info', pr=None): - """Add a message to message queue in communicator object. - - The queued message is constructed based on message string passed as - a method argument and message header. Message header is mapped to message severity - also passed as an argument. - - :param message: Message content - :param message_severity: Message severity level - :type message: String - :type message_severity: int - """ - log.info(message) - internal = False - if 'internal' in message_severity: - message_header = INTERNAL_ERROR_MESSAGE_HEADER - internal = True - elif 'error' in message_severity: - message_header = ERROR_MESSAGE_HEADER - elif 'warning' in message_severity: - message_header = WARNING_MESSAGE_HEADER - else: - message_header = INFO_MESSAGE_HEADER - # If message is related to PR attatch url - if pr: - message = message + '\n' + pr.html_url - - send = message_header + '\n' + message - if self._ms_teams_enabled: - self._msteams_hook.queue_message(send) - - def _check_finished(self, pr, build_number): - """Verify if finished build output contains expected string for either fail or success. - - :param pr: Single PR being currently checked - :param build_number: Jenkins CI job build number - :type pr: github.PullRequest.PullRequest - :type build_number: int - """ - pr_number = str(pr.number) - log.info('CI for PR %s: FINISHED', pr_number) - # Check if FINISH was valid FAIL / SUCCESS - project_name_full = self._ci_job_name + '/PR-' + pr_number - build_output = self._jenkins.get_build_console_output(project_name_full, build_number) - if _CI_BUILD_FAIL_MESSAGE not in build_output \ - and _CI_BUILD_SUCCESS_MESSAGE not in build_output: - message = ('ONNX CI job for PR #{}: finished but no tests success or fail ' - 'confirmation is present in console output!'.format(pr_number)) - self._queue_message(message, message_severity='error', pr=pr) - - def _send_message(self, quiet=False): - """Send messages queued in MS Teams objects to designated channel. - - Queued messages are being sent as a single communication. - - :param quiet: Flag for disabling sending report through communicator - :type quiet: Boolean - """ - if any(messages for messages in self._msteams_hook.messages): - try: - watchdog_build = self._jenkins.get_job_info(self._watchdog_job_name)['lastBuild'] - watchdog_build_number = watchdog_build['number'] - watchdog_build_link = watchdog_build['url'] - except Exception: - watchdog_build_number = 'UNKNOWN' - watchdog_build_link = self._jenkins.jenkins_server - send = self._watchdog_job_name + '- build ' + str( - watchdog_build_number) + ' - ' + watchdog_build_link - - if self._ms_teams_enabled: - self._msteams_hook.send_message(send, quiet=quiet) - else: - log.info('Nothing to report.') - - def _check_in_progress(self, pr, build_number): - """Check if CI build succesfully started. - - Checks if build started within designated time threshold, and job is - currently running - it didn't cross the time threshold. - - :param pr: Single PR being currently checked - :param build_number: Jenkins CI job build number - :type pr: github.PullRequest.PullRequest - :type build_number: int - """ - pr_number = str(pr.number) - log.info('CI for PR %s: TESTING IN PROGRESS', pr_number) - project_name_full = self._ci_job_name + '/PR-' + pr_number - build_info = self._jenkins.get_build_info(project_name_full, build_number) - build_datetime = datetime.datetime.fromtimestamp(build_info['timestamp'] / 1000.0) - build_delta = self._now_time - build_datetime - log.info('Build %s: IN PROGRESS, started: %s minutes ago', str(build_number), - str(build_delta)) - # If build still waiting in queue - if build_delta > _CI_START_THRESHOLD and self._build_in_queue(pr, build_number): - message = ('ONNX CI job build #{}, for PR #{} waiting in queue after {} ' - 'minutes'.format(build_number, pr_number, str(build_delta.seconds / 60))) - self._queue_message(message, message_severity='warning', pr=pr) - elif build_delta > _BUILD_DURATION_THRESHOLD: - # CI job take too long, possibly froze - communicate failure - message = ('ONNX CI job build #{}, for PR #{} started, ' - 'but did not finish in designated time of {} ' - 'minutes!'.format(build_number, pr_number, - str(_BUILD_DURATION_THRESHOLD.seconds / 60))) - self._queue_message(message, message_severity='error', pr=pr) - - def _update_config(self): - """Update Watchdog config file with PRs checked in current Watchdog run, remove old entries. - - :param current_prs: List of PR numbers checked during current Watchdog run - :type current_prs: list of ints - """ - # Cleanup config of old reports - log.info('Writing to config file at: {}'.format(self._config_path)) - new_config = {_PR_REPORTS_CONFIG_KEY: self._current_prs} - file = open(self._config_path, 'w+') - json.dump(new_config, file) diff --git a/.github/workflows/py_checks.yml b/.github/workflows/py_checks.yml index 6cec72ac5e6..379492516e7 100644 --- a/.github/workflows/py_checks.yml +++ b/.github/workflows/py_checks.yml @@ -1,4 +1,4 @@ -name: IE Python Checks +name: Python API Checks on: workflow_dispatch: @@ -23,8 +23,9 @@ jobs: with: python-version: '3.6' - name: Install dependencies - run: python -m pip install -r src/bindings/python/src/compatibility/openvino/requirements_dev.txt - - name: Run Flake on samples + run: python -m pip install -r src/bindings/python/requirements_test.txt + # samples code-style + - name: Run flake8 on samples run: python -m flake8 ./ --config=setup.cfg working-directory: samples/python - name: Create code style diff for samples @@ -38,21 +39,53 @@ jobs: with: name: samples_diff path: samples_diff.diff - - name: Run Flake on src + # IE Python API Flake code-style + - name: Run flake8 on IE Python API run: python -m flake8 ./ --config=setup.cfg working-directory: src/bindings/python/src/compatibility/openvino - - name: Create code style diff for Python src + - name: Create code style diff for IE Python API if: failure() run: | python -m black -l 160 -S ./ - git diff > src_diff.diff + git diff > ie_python_diff.diff working-directory: src/bindings/python/src/compatibility/openvino - uses: actions/upload-artifact@v2 if: failure() with: - name: src_diff - path: src_diff.diff - - name: Run Flake on wheel + name: ie_python_diff + path: ie_python_diff.diff + # nGraph Python API Flake code-style + - name: Run flake8 on nGraph Python API + run: python -m flake8 ./src/compatibility/ngraph --config=setup.cfg + working-directory: src/bindings/python + - name: Create code style diff for nGraph Python API + if: failure() + run: | + python -m black -l 160 -S ./ + git diff > pyngraph_diff.diff + working-directory: src/bindings/python/src/compatibility/ngraph + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: pyngraph_diff + path: pyngraph_diff.diff + # Python API 2.0 Flake code-style + - name: Run flake8 on Python API 2.0 + run: python -m flake8 ./src/openvino --config=setup.cfg + working-directory: src/bindings/python + - name: Create code style diff for Python API 2.0 + if: failure() + run: | + python -m black -l 160 -S ./ + git diff > pyopenvino_diff.diff + working-directory: src/bindings/python/src/openvino + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: pyopenvino_diff + path: pyopenvino_diff.diff + # wheel Flake code-style + - name: Run flake8 on wheel run: python -m flake8 ./ --config=../setup.cfg working-directory: src/bindings/python/wheel - name: Create code style diff for wheel @@ -66,10 +99,24 @@ jobs: with: name: wheel_diff path: wheel_diff.diff - - - name: Run MyPy + # Python API 2.0 tests Flake code-style + - name: Run flake8 on python tests + # ignore lack of docs in tests + run: python -m flake8 tests/ --config=setup.cfg + working-directory: src/bindings/python + # IE Python API mypy check + - name: Run mypy on IE Python API run: python -m mypy ./ --config-file ./setup.cfg working-directory: src/bindings/python/src/compatibility/openvino + # nGraph Python API mypy check + - name: Run mypy on nGraph Python API + run: python -m mypy ./src/compatibility/ngraph --config-file ./setup.cfg + working-directory: src/bindings/python + # Python API 2.0 mypy check + - name: Run mypy on Python API 2.0 + run: python -m mypy ./src/openvino --config-file ./setup.cfg + working-directory: src/bindings/python + - name: Run Bandit run: python -m bandit -r ./ -f screen working-directory: src/bindings/python/src/compatibility/openvino diff --git a/samples/python/classification_sample_async/classification_sample_async.py b/samples/python/classification_sample_async/classification_sample_async.py index 3cdbb512b0d..ed8ee93cf12 100755 --- a/samples/python/classification_sample_async/classification_sample_async.py +++ b/samples/python/classification_sample_async/classification_sample_async.py @@ -14,7 +14,7 @@ from openvino.runtime import AsyncInferQueue, Core, InferRequest, Layout, Type def parse_args() -> argparse.Namespace: - """Parse and return command line arguments""" + """Parse and return command line arguments.""" parser = argparse.ArgumentParser(add_help=False) args = parser.add_argument_group('Options') # fmt: off diff --git a/samples/python/hello_query_device/hello_query_device.py b/samples/python/hello_query_device/hello_query_device.py index c9a7ccd50cd..0bf37f55edc 100755 --- a/samples/python/hello_query_device/hello_query_device.py +++ b/samples/python/hello_query_device/hello_query_device.py @@ -9,7 +9,7 @@ from openvino.runtime import Core def param_to_string(parameters) -> str: - """Convert a list / tuple of parameters returned from IE to a string""" + """Convert a list / tuple of parameters returned from IE to a string.""" if isinstance(parameters, (list, tuple)): return ', '.join([str(x) for x in parameters]) else: diff --git a/samples/python/model_creation_sample/model_creation_sample.py b/samples/python/model_creation_sample/model_creation_sample.py index 33b796d83da..6f4845eac66 100755 --- a/samples/python/model_creation_sample/model_creation_sample.py +++ b/samples/python/model_creation_sample/model_creation_sample.py @@ -16,7 +16,7 @@ from data import digits def create_ngraph_function(model_path: str) -> Model: - """Create a model on the fly from the source code using ngraph""" + """Create a model on the fly from the source code using ngraph.""" def shape_and_length(shape: list) -> typing.Tuple[list, int]: length = reduce(lambda x, y: x * y, shape) diff --git a/samples/python/setup.cfg b/samples/python/setup.cfg index f16854c1641..1e8c86607e1 100644 --- a/samples/python/setup.cfg +++ b/samples/python/setup.cfg @@ -1,7 +1,11 @@ [flake8] +# ignore: +# D100 - Missing docstring in public module +# D101 - Missing docstring in public class +# D103 - Missing docstring in public function filename = *.py max-line-length = 160 -ignore = E203 +ignore = E203,D100,D101,D103 max-parameters-amount = 8 show_source = True docstring-convention = google diff --git a/samples/python/speech_sample/arg_parser.py b/samples/python/speech_sample/arg_parser.py index 2cf0e9f7545..7c30a801d64 100644 --- a/samples/python/speech_sample/arg_parser.py +++ b/samples/python/speech_sample/arg_parser.py @@ -5,7 +5,7 @@ import argparse def build_arg_parser() -> argparse.ArgumentParser: - """Create and return argument parser""" + """Create and return argument parser.""" parser = argparse.ArgumentParser(add_help=False) args = parser.add_argument_group('Options') model = parser.add_mutually_exclusive_group(required=True) @@ -73,7 +73,7 @@ def build_arg_parser() -> argparse.ArgumentParser: def parse_args() -> argparse.Namespace: - """Parse and validate command-line arguments""" + """Parse and validate command-line arguments.""" parser = build_arg_parser() args = parser.parse_args() diff --git a/samples/python/speech_sample/file_options.py b/samples/python/speech_sample/file_options.py index 0d21294ea73..a0203e695f0 100644 --- a/samples/python/speech_sample/file_options.py +++ b/samples/python/speech_sample/file_options.py @@ -15,9 +15,9 @@ class FileData(NamedTuple): def read_ark_file(file_name: str) -> FileData: - """Read utterance matrices from a .ark file""" + """Read utterance matrices from a .ark file.""" def read_key(input_file: IO[Any]) -> str: - """Read a identifier of utterance matrix""" + """Read a identifier of utterance matrix.""" key = '' char = input_file.read(1).decode() @@ -28,7 +28,7 @@ def read_ark_file(file_name: str) -> FileData: return key def read_matrix(input_file: IO[Any]) -> np.ndarray: - """Read a utterance matrix""" + """Read a utterance matrix.""" header = input_file.read(5).decode() if 'FM' in header: num_of_bytes = 4 @@ -61,7 +61,7 @@ def read_ark_file(file_name: str) -> FileData: def write_ark_file(file_name: str, keys: List[str], utterances: List[np.ndarray]): - """Write utterance matrices to a .ark file""" + """Write utterance matrices to a .ark file.""" with open(file_name, 'wb') as output_file: for key, matrix in zip(keys, utterances): # write a utterance key @@ -86,7 +86,7 @@ def write_ark_file(file_name: str, keys: List[str], utterances: List[np.ndarray] def read_utterance_file(file_name: str) -> FileData: - """Read utterance matrices from a file""" + """Read utterance matrices from a file.""" file_extension = file_name.split('.')[-1] if file_extension == 'ark': @@ -100,7 +100,7 @@ def read_utterance_file(file_name: str) -> FileData: def write_utterance_file(file_name: str, keys: List[str], utterances: List[np.ndarray]): - """Write utterance matrices to a file""" + """Write utterance matrices to a file.""" file_extension = file_name.split('.')[-1] if file_extension == 'ark': diff --git a/samples/python/speech_sample/speech_sample.py b/samples/python/speech_sample/speech_sample.py index 9b71e55aca3..1b708c4a3f0 100755 --- a/samples/python/speech_sample/speech_sample.py +++ b/samples/python/speech_sample/speech_sample.py @@ -22,7 +22,7 @@ from utils import (GNA_ATOM_FREQUENCY, GNA_CORE_FREQUENCY, def do_inference(data: Dict[str, np.ndarray], infer_request: InferRequest, cw_l: int = 0, cw_r: int = 0) -> np.ndarray: - """Do a synchronous matrix inference""" + """Do a synchronous matrix inference.""" frames_to_infer = {} result = {} diff --git a/samples/python/speech_sample/utils.py b/samples/python/speech_sample/utils.py index e3a87635958..86980070cb3 100644 --- a/samples/python/speech_sample/utils.py +++ b/samples/python/speech_sample/utils.py @@ -35,7 +35,7 @@ def compare_with_reference(result: np.ndarray, reference: np.ndarray): def get_scale_factor(matrix: np.ndarray) -> float: - """Get scale factor for quantization using utterance matrix""" + """Get scale factor for quantization using utterance matrix.""" # Max to find scale factor target_max = 16384 max_val = np.max(matrix) @@ -46,14 +46,14 @@ def get_scale_factor(matrix: np.ndarray) -> float: def set_scale_factors(plugin_config: dict, scale_factors: list): - """Set a scale factor provided for each input""" + """Set a scale factor provided for each input.""" for i, scale_factor in enumerate(scale_factors): log.info(f'For input {i} using scale factor of {scale_factor:.7f}') plugin_config[f'GNA_SCALE_FACTOR_{i}'] = str(scale_factor) def parse_scale_factors(args: argparse.Namespace) -> list: - """Get a list of scale factors for input files""" + """Get a list of scale factors for input files.""" input_files = re.split(', |,', args.input) scale_factors = re.split(', |,', str(args.scale_factor)) scale_factors = list(map(float, scale_factors)) @@ -72,7 +72,7 @@ def parse_scale_factors(args: argparse.Namespace) -> list: def parse_outputs_from_args(args: argparse.Namespace) -> Tuple[List[str], List[int]]: - """Get a list of outputs specified in the args""" + """Get a list of outputs specified in the args.""" name_and_port = [output.split(':') for output in re.split(', |,', args.output_layers)] try: return [name for name, _ in name_and_port], [int(port) for _, port in name_and_port] diff --git a/src/bindings/python/requirements_test.txt b/src/bindings/python/requirements_test.txt index f8174c2607b..ecac2cf0bb5 100644 --- a/src/bindings/python/requirements_test.txt +++ b/src/bindings/python/requirements_test.txt @@ -1,13 +1,35 @@ -flake8==3.9.2 -flake8-bugbear==21.4.3 -flake8-comprehensions==3.3.0 -flake8-docstrings==1.6.0 -flake8-quotes==3.2.0 -mypy==0.812 -onnx==1.11.0 -pydocstyle==5.1.1 -pytest-xdist==2.2.1 -pytest==6.1.2 -retrying==1.3.3 -tox==3.24.5 -wheel==0.36.2 +bandit +black +flake8 +flake8-annotations-complexity +flake8-broken-line +flake8-bugbear +flake8-class-attributes-order +flake8-comprehensions +flake8-debugger +flake8-docstrings +flake8-eradicate +flake8-executable +flake8-expression-complexity +flake8-print +flake8-pytest-style +flake8-rst-docstrings +flake8-string-format +flake8-variables-names +flake8_builtins +flake8_coding +flake8_commas +flake8_pep3101 +flake8_quotes +import-order +mypy +onnx +Pep8-naming +pydocstyle +pytest-xdist +pytest +radon +retrying +tox +types-pkg_resources +wheel \ No newline at end of file diff --git a/src/bindings/python/setup.cfg b/src/bindings/python/setup.cfg index cf1949d0869..90e9b6d8fac 100644 --- a/src/bindings/python/setup.cfg +++ b/src/bindings/python/setup.cfg @@ -1,15 +1,62 @@ +[tox:tox] +envlist = py3 + +[testenv] +skipdist=True +skip_install=True +deps = + -rrequirements.txt + -rrequirements_test.txt +setenv = + OV_BACKEND = {env:OV_BACKEND:"CPU"} + PYTHONPATH = {env:PYTHONPATH} + OpenVINO_DIR = {env:OpenVINO_DIR} +passenv = + http_proxy + https_proxy +commands= + {envbindir}/python setup.py bdist_wheel + {envbindir}/pip install --no-index --pre --find-links=dist/ openvino + pytest --backend={env:OV_BACKEND} tests -v -k 'not _cuda' --ignore=tests/test_onnx/test_zoo_models.py --ignore=tests/test_utils --ignore=tests/test_inference_engine + pytest --backend={env:OV_BACKEND} tests_compatibility/test_ngraph -v -k 'not _cuda' --ignore=tests_compatibility/test_onnx/test_zoo_models.py + +[testenv:zoo_models] +commands= + {envbindir}/python setup.py bdist_wheel + {envbindir}/pip install --no-index --pre --find-links=dist/ openvino + pytest --backend={env:OV_BACKEND} tests/test_onnx/test_zoo_models.py -v -n 4 --forked -k 'not _cuda' --model_zoo_xfail + +[testenv:devenv] +envdir = devenv +usedevelop = True +deps = -rrequirements.txt + [flake8] -filename = *.py, *.pyx +# ignore: +# D100 - Missing docstring in public module +# D102 - Missing docstring in public method +# D103 - Missing docstring in public function +# D104 - Missing docstring in public package +# D105 - Missing docstring in magic method +# D107 - Missing docstring in __init__ +# D412 - No blank lines allowed between a section header and its content +# F401 - module imported but unused +# W503 - line break before binary operator (prefer line breaks before op, not after) +ignore=D100,D102,D103,D104,D105,D107,D412,F401,W503 +inline-quotes = double +filename = *.py max-line-length = 160 -ignore = E203 max-parameters-amount = 8 show_source = True docstring-convention = google enable-extensions = G per-file-ignores = - *.pyx: E225, E226, E251, E999, E800, E265, E203, E266, E227, E211 - tests/*: S101, T001 - *__init__.py: E402, F403, F405, F405 + # todo: will be fixed as part of 81803 + tests/*: A001,C101,C812,C815,C816,C819,CCE001,D100,D101,D102,D103,D104,D105,D107,D212,E800,ECE001,N400,N802,N806,P101,P103,PT001,PT005,PT006,PT007,PT011,PT012,PT019,PT023,RST201,RST301,S001,T001,VNE001,VNE002,VNE003,W503 + tests_compatibility/test_ngraph/*: A001,C101,C812,C815,C816,C819,CCE001,D100,D101,D102,D103,D104,D105,D107,D212,E800,ECE001,N400,N802,N806,P101,P103,PT001,PT005,PT006,PT007,PT011,PT012,PT019,PT023,RST201,RST301,S001,T001,VNE001,VNE002,VNE003,W503 + src/compatibility/ngraph/*: A001,A002,C101,C812,C819,CCE001,E800,N803,N806,P101,RST201,RST202,RST203,RST206,RST301,T001,TAE002,VNE001,VNE003 + # todo: will be fixed as part of 81803 + src/openvino/*: A001,A002,C101,C812,C819,CCE001,E402,E800,N803,N806,P101,RST201,RST202,RST203,RST205,RST206,RST299,RST301,T001,TAE002,VNE001,VNE002,VNE003 [pydocstyle] convention = google @@ -20,4 +67,9 @@ disable_error_code = attr-defined show_column_numbers = True show_error_context = True show_absolute_path = True -pretty = True \ No newline at end of file +pretty = True +follow_imports=normal +disallow_untyped_defs = True +disallow_untyped_calls = True +check_untyped_defs = True +show_none_errors = True diff --git a/src/bindings/python/src/compatibility/openvino/setup.cfg b/src/bindings/python/src/compatibility/openvino/setup.cfg index ddfd39a5831..fc7f6012f70 100644 --- a/src/bindings/python/src/compatibility/openvino/setup.cfg +++ b/src/bindings/python/src/compatibility/openvino/setup.cfg @@ -1,16 +1,16 @@ [flake8] +# D104 - Missing docstring in public package inline-quotes = double filename = *.py, *.pyx max-line-length = 160 -ignore = E203 +ignore = E203,D104 max-parameters-amount = 8 show_source = True docstring-convention = google enable-extensions = G per-file-ignores = *.pyx: E225, E226, E251, E999, E800, E265, E203, E266, E227, E211 - tests/*: S101, T001 - *__init__.py: F403, F405, F405 + *__init__.py: F403, F405 [pydocstyle] convention = google diff --git a/src/bindings/python/tox.ini b/src/bindings/python/tox.ini deleted file mode 100644 index bd9568bf07d..00000000000 --- a/src/bindings/python/tox.ini +++ /dev/null @@ -1,75 +0,0 @@ -[tox] -envlist = py3 - -[testenv] -skipdist=True -skip_install=True -deps = - -rrequirements.txt - -rrequirements_test.txt -setenv = - NGRAPH_BACKEND = {env:NGRAPH_BACKEND:"CPU"} - PYTHONPATH = {env:PYTHONPATH} - OpenVINO_DIR = {env:OpenVINO_DIR} -passenv = - http_proxy - https_proxy -commands= - {envbindir}/python setup.py bdist_wheel - {envbindir}/pip install --no-index --pre --find-links=dist/ openvino - flake8 {posargs:src/ setup.py} --exclude=src/compatibility/openvino - flake8 --ignore=D100,D101,D102,D103,D104,D105,D107,D212,W503 tests/ tests_compatibility/ --exclude=tests_compatibility/test_inference_engine # ignore lack of docs in tests - mypy --config-file=tox.ini {posargs:src/} --exclude=src/compatibility/openvino - pytest --backend={env:NGRAPH_BACKEND} tests -v -k 'not _cuda' --ignore=tests/test_onnx/test_zoo_models.py --ignore=tests/test_utils --ignore=tests/test_inference_engine - pytest --backend={env:NGRAPH_BACKEND} tests_compatibility/test_ngraph -v -k 'not _cuda' --ignore=tests_compatibility/test_onnx/test_zoo_models.py - -[testenv:zoo_models] -commands= - {envbindir}/python setup.py bdist_wheel - {envbindir}/pip install --no-index --pre --find-links=dist/ openvino - pytest --backend={env:NGRAPH_BACKEND} tests/test_onnx/test_zoo_models.py -v -n 4 --forked -k 'not _cuda' --model_zoo_xfail - -[testenv:devenv] -envdir = devenv -usedevelop = True -deps = -rrequirements.txt - -[flake8] -inline-quotes = " -max-line-length=110 -max-complexity=7 -# ignore: -# D100 - Missing docstring in public module -# D104 - Missing docstring in public package -# D105 - Missing docstring in magic method -# D107 - Missing docstring in __init__ -# D412 - No blank lines allowed between a section header and its content -# F401 - module imported but unused -# W503 - line break before binary operator (prefer line breaks before op, not after) -ignore=D100,D104,D105,D107,D410,D411,D412,F401,W503 - -[mypy] -ignore_missing_imports=True -follow_imports=normal -disallow_untyped_defs = True -disallow_untyped_calls = True -check_untyped_defs = True -show_error_context = True -show_column_numbers = True -show_none_errors = True - -# put custom per-file options here in sections that map their names into filenames, e.g. gta_workflow/filename.py is -# [mypy-ngraph/filename] - -[mypy-test.*] -disallow_untyped_defs = False - -[isort] -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -use_parentheses=True -line_length=100 -default_section=FIRSTPARTY -sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER -known_third_party=openvino diff --git a/src/bindings/python/wheel/setup.py b/src/bindings/python/wheel/setup.py index e3ff90622fb..5b070c269b7 100644 --- a/src/bindings/python/wheel/setup.py +++ b/src/bindings/python/wheel/setup.py @@ -22,118 +22,118 @@ from setuptools.command.build_ext import build_ext from setuptools.command.build_clib import build_clib from setuptools.command.install import install -WHEEL_LIBS_INSTALL_DIR = os.path.join('openvino', 'libs') -WHEEL_LIBS_PACKAGE = 'openvino.libs' -PYTHON_VERSION = f'python{sys.version_info.major}.{sys.version_info.minor}' +WHEEL_LIBS_INSTALL_DIR = os.path.join("openvino", "libs") +WHEEL_LIBS_PACKAGE = "openvino.libs" +PYTHON_VERSION = f"python{sys.version_info.major}.{sys.version_info.minor}" -LIBS_DIR = 'bin' if platform.system() == 'Windows' else 'lib' -CONFIG = 'Release' if platform.system() in {'Windows' , 'Darwin'} else '' +LIBS_DIR = "bin" if platform.system() == "Windows" else "lib" +CONFIG = "Release" if platform.system() in {"Windows", "Darwin"} else "" machine = platform.machine() -if machine == 'x86_64' or machine == 'AMD64': - ARCH = 'intel64' -elif machine == 'X86': - ARCH = 'ia32' -elif machine == 'arm': - ARCH = 'arm' -elif machine == 'aarch64': - ARCH = 'arm64' +if machine == "x86_64" or machine == "AMD64": + ARCH = "intel64" +elif machine == "X86": + ARCH = "ia32" +elif machine == "arm": + ARCH = "arm" +elif machine == "aarch64": + ARCH = "arm64" # The following variables can be defined in environment or .env file SCRIPT_DIR = Path(__file__).resolve().parents[0] -CMAKE_BUILD_DIR = os.getenv('CMAKE_BUILD_DIR', '.') -OV_RUNTIME_LIBS_DIR = os.getenv('OV_RUNTIME_LIBS_DIR', f'runtime/{LIBS_DIR}/{ARCH}/{CONFIG}') -TBB_LIBS_DIR = os.getenv('TBB_LIBS_DIR', f'runtime/3rdparty/tbb/{LIBS_DIR}') -PY_PACKAGES_DIR = os.getenv('PY_PACKAGES_DIR', f'python/{PYTHON_VERSION}') -LIBS_RPATH = '$ORIGIN' if sys.platform == 'linux' else '@loader_path' +CMAKE_BUILD_DIR = os.getenv("CMAKE_BUILD_DIR", ".") +OV_RUNTIME_LIBS_DIR = os.getenv("OV_RUNTIME_LIBS_DIR", f"runtime/{LIBS_DIR}/{ARCH}/{CONFIG}") +TBB_LIBS_DIR = os.getenv("TBB_LIBS_DIR", f"runtime/3rdparty/tbb/{LIBS_DIR}") +PY_PACKAGES_DIR = os.getenv("PY_PACKAGES_DIR", f"python/{PYTHON_VERSION}") +LIBS_RPATH = "$ORIGIN" if sys.platform == "linux" else "@loader_path" LIB_INSTALL_CFG = { - 'ie_libs': { - 'name': 'core', - 'prefix': 'libs.core', - 'install_dir': OV_RUNTIME_LIBS_DIR, - 'rpath': LIBS_RPATH, + "ie_libs": { + "name": "core", + "prefix": "libs.core", + "install_dir": OV_RUNTIME_LIBS_DIR, + "rpath": LIBS_RPATH, }, - 'hetero_plugin': { - 'name': 'hetero', - 'prefix': 'libs.core', - 'install_dir': OV_RUNTIME_LIBS_DIR, - 'rpath': LIBS_RPATH, + "hetero_plugin": { + "name": "hetero", + "prefix": "libs.core", + "install_dir": OV_RUNTIME_LIBS_DIR, + "rpath": LIBS_RPATH, }, - 'gpu_plugin': { - 'name': 'gpu', - 'prefix': 'libs.core', - 'install_dir': OV_RUNTIME_LIBS_DIR, - 'rpath': LIBS_RPATH, + "gpu_plugin": { + "name": "gpu", + "prefix": "libs.core", + "install_dir": OV_RUNTIME_LIBS_DIR, + "rpath": LIBS_RPATH, }, - 'cpu_plugin': { - 'name': 'cpu', - 'prefix': 'libs.core', - 'install_dir': OV_RUNTIME_LIBS_DIR, - 'rpath': LIBS_RPATH, + "cpu_plugin": { + "name": "cpu", + "prefix": "libs.core", + "install_dir": OV_RUNTIME_LIBS_DIR, + "rpath": LIBS_RPATH, }, - 'multi_plugin': { - 'name': 'multi', - 'prefix': 'libs.core', - 'install_dir': OV_RUNTIME_LIBS_DIR, - 'rpath': LIBS_RPATH, + "multi_plugin": { + "name": "multi", + "prefix": "libs.core", + "install_dir": OV_RUNTIME_LIBS_DIR, + "rpath": LIBS_RPATH, }, - 'batch_plugin': { - 'name': 'batch', - 'prefix': 'libs.core', - 'install_dir': OV_RUNTIME_LIBS_DIR, + "batch_plugin": { + "name": "batch", + "prefix": "libs.core", + "install_dir": OV_RUNTIME_LIBS_DIR, }, - 'tbb_libs': { - 'name': 'tbb', - 'prefix': 'libs.tbb', - 'install_dir': TBB_LIBS_DIR, - 'rpath': LIBS_RPATH, + "tbb_libs": { + "name": "tbb", + "prefix": "libs.tbb", + "install_dir": TBB_LIBS_DIR, + "rpath": LIBS_RPATH, }, } PY_INSTALL_CFG = { - 'ie_py': { - 'name': PYTHON_VERSION, - 'prefix': 'site-packages', - 'install_dir': PY_PACKAGES_DIR, + "ie_py": { + "name": PYTHON_VERSION, + "prefix": "site-packages", + "install_dir": PY_PACKAGES_DIR, }, - 'ngraph_py': { - 'name': f'pyngraph_{PYTHON_VERSION}', - 'prefix': 'site-packages', - 'install_dir': PY_PACKAGES_DIR, + "ngraph_py": { + "name": f"pyngraph_{PYTHON_VERSION}", + "prefix": "site-packages", + "install_dir": PY_PACKAGES_DIR, }, - 'pyopenvino' : { - 'name': f'pyopenvino_{PYTHON_VERSION}', - 'prefix': 'site-packages', - 'install_dir': PY_PACKAGES_DIR, + "pyopenvino": { + "name": f"pyopenvino_{PYTHON_VERSION}", + "prefix": "site-packages", + "install_dir": PY_PACKAGES_DIR, }, } class PrebuiltExtension(Extension): - """Initialize Extension""" + """Initialize Extension.""" def __init__(self, name, sources, *args, **kwargs): if len(sources) != 1: - nln = '\n' - raise DistutilsSetupError(f'PrebuiltExtension can accept only one source, but got: {nln}{nln.join(sources)}') + nln = "\n" + raise DistutilsSetupError(f"PrebuiltExtension can accept only one source, but got: {nln}{nln.join(sources)}") super().__init__(name, sources, *args, **kwargs) class CustomBuild(build): - """Custom implementation of build_clib""" + """Custom implementation of build_clib.""" - cmake_build_types = ['Release', 'Debug', 'RelWithDebInfo', 'MinSizeRel'] + cmake_build_types = ["Release", "Debug", "RelWithDebInfo", "MinSizeRel"] user_options = [ - ('config=', None, 'Build configuration [{types}].'.format(types='|'.join(cmake_build_types))), - ('jobs=', None, 'Specifies the number of jobs to use with make.'), - ('cmake-args=', None, 'Additional options to be passed to CMake.'), + ("config=", None, "Build configuration [{types}].".format(types="|".join(cmake_build_types))), + ("jobs=", None, "Specifies the number of jobs to use with make."), + ("cmake-args=", None, "Additional options to be passed to CMake."), ] def initialize_options(self): """Set default values for all the options that this command supports.""" super().initialize_options() - self.build_base = 'build' + self.build_base = "build" self.config = None self.jobs = None self.cmake_args = None @@ -144,52 +144,52 @@ class CustomBuild(build): if not self.config: if self.debug: - self.config = 'Debug' + self.config = "Debug" else: - self.announce('Set default value for CMAKE_BUILD_TYPE = Release.', level=4) - self.config = 'Release' + self.announce("Set default value for CMAKE_BUILD_TYPE = Release.", level=4) + self.config = "Release" else: build_types = [item.lower() for item in self.cmake_build_types] try: i = build_types.index(str(self.config).lower()) self.config = self.cmake_build_types[i] - self.debug = True if 'Debug' == self.config else False + self.debug = True if "Debug" == self.config else False except ValueError: - self.announce('Unsupported CMAKE_BUILD_TYPE value: ' + self.config, level=4) - self.announce('Supported values: {types}'.format(types=', '.join(self.cmake_build_types)), level=4) + self.announce("Unsupported CMAKE_BUILD_TYPE value: " + self.config, level=4) + self.announce("Supported values: {types}".format(types=", ".join(self.cmake_build_types)), level=4) sys.exit(1) - if self.jobs is None and os.getenv('MAX_JOBS') is not None: - self.jobs = os.getenv('MAX_JOBS') + if self.jobs is None and os.getenv("MAX_JOBS") is not None: + self.jobs = os.getenv("MAX_JOBS") self.jobs = multiprocessing.cpu_count() if self.jobs is None else int(self.jobs) def run(self): global CMAKE_BUILD_DIR self.jobs = multiprocessing.cpu_count() - plat_specifier = '.{0}-{1}.{2}'.format(self.plat_name, *sys.version_info[:2]) - self.build_temp = os.path.join(self.build_base, 'temp' + plat_specifier, self.config) + plat_specifier = ".{0}-{1}.{2}".format(self.plat_name, *sys.version_info[:2]) + self.build_temp = os.path.join(self.build_base, "temp" + plat_specifier, self.config) # if setup.py is directly called use CMake to build product - if CMAKE_BUILD_DIR == '.': + if CMAKE_BUILD_DIR == ".": # set path to the root of OpenVINO CMakeList file openvino_root_dir = Path(__file__).resolve().parents[4] - self.announce(f'Configuring cmake project: {openvino_root_dir}', level=3) - self.spawn(['cmake', '-H' + str(openvino_root_dir), '-B' + self.build_temp, - '-DCMAKE_BUILD_TYPE={type}'.format(type=self.config), - '-DENABLE_PYTHON=ON', - '-DENABLE_OV_ONNX_FRONTEND=ON']) + self.announce(f"Configuring cmake project: {openvino_root_dir}", level=3) + self.spawn(["cmake", "-H" + str(openvino_root_dir), "-B" + self.build_temp, + "-DCMAKE_BUILD_TYPE={type}".format(type=self.config), + "-DENABLE_PYTHON=ON", + "-DENABLE_OV_ONNX_FRONTEND=ON"]) - self.announce('Building binaries', level=3) - self.spawn(['cmake', '--build', self.build_temp, - '--config', self.config, '-j', str(self.jobs)]) + self.announce("Building binaries", level=3) + self.spawn(["cmake", "--build", self.build_temp, + "--config", self.config, "-j", str(self.jobs)]) CMAKE_BUILD_DIR = self.build_temp - self.run_command('build_clib') + self.run_command("build_clib") build.run(self) # Copy extra package_data content filtered by find_packages dst = Path(self.build_lib) src = Path(get_package_dir(PY_INSTALL_CFG)) - exclude = ignore_patterns('*ez_setup*', '*__pycache__*', '*.egg-info*') - for path in src.glob('**/*'): + exclude = ignore_patterns("*ez_setup*", "*__pycache__*", "*.egg-info*") + for path in src.glob("**/*"): if path.is_dir() or exclude(str(path)): continue path_rel = path.relative_to(src) @@ -198,7 +198,7 @@ class CustomBuild(build): class PrepareLibs(build_clib): - """Prepare prebuilt libraries""" + """Prepare prebuilt libraries.""" def run(self): self.configure(LIB_INSTALL_CFG) @@ -208,30 +208,27 @@ class PrepareLibs(build_clib): def configure(self, install_cfg): """Collect prebuilt libraries. Install them to the temp directories, set rpath.""" for comp, comp_data in install_cfg.items(): - install_prefix = comp_data.get('prefix') - install_dir = comp_data.get('install_dir') + install_prefix = comp_data.get("prefix") + install_dir = comp_data.get("install_dir") if install_dir and not os.path.isabs(install_dir): install_dir = os.path.join(install_prefix, install_dir) - self.announce(f'Installing {comp}', level=3) - self.spawn(['cmake', '--install', CMAKE_BUILD_DIR, '--prefix', install_prefix, '--component', comp_data.get('name')]) + self.announce(f"Installing {comp}", level=3) + self.spawn(["cmake", "--install", CMAKE_BUILD_DIR, "--prefix", install_prefix, "--component", comp_data.get("name")]) # set rpath if applicable - if sys.platform != 'win32' and comp_data.get('rpath'): - file_types = ['.so'] if sys.platform == 'linux' else ['.dylib', '.so'] - for path in filter(lambda p: any(item in file_types for item in p.suffixes), Path(install_dir).glob('*')): - set_rpath(comp_data['rpath'], os.path.realpath(path)) + if sys.platform != "win32" and comp_data.get("rpath"): + file_types = [".so"] if sys.platform == "linux" else [".dylib", ".so"] + for path in filter(lambda p: any(item in file_types for item in p.suffixes), Path(install_dir).glob("*")): + set_rpath(comp_data["rpath"], os.path.realpath(path)) def generate_package(self, src_dirs): - """ - Collect package data files from preinstalled dirs and - put all runtime libraries to the subpackage - """ + """Collect package data files from preinstalled dirs and put all runtime libraries to the subpackage.""" # additional blacklist filter, just to fix cmake install issues - blacklist = ['.lib', '.pdb', '_debug.dll', '_debug.dylib'] + blacklist = [".lib", ".pdb", "_debug.dll", "_debug.dylib"] package_dir = os.path.join(get_package_dir(PY_INSTALL_CFG), WHEEL_LIBS_INSTALL_DIR) for src_dir in src_dirs: local_base_dir = Path(src_dir) - for file_path in local_base_dir.rglob('*'): + for file_path in local_base_dir.rglob("*"): file_name = os.path.basename(file_path) if file_path.is_file() and not any(file_name.endswith(ext) for ext in blacklist): dst_file = os.path.join(package_dir, os.path.relpath(file_path, local_base_dir)) @@ -239,51 +236,51 @@ class PrepareLibs(build_clib): copyfile(file_path, dst_file) if Path(package_dir).exists(): - self.announce(f'Adding {WHEEL_LIBS_PACKAGE} package', level=3) + self.announce(f"Adding {WHEEL_LIBS_PACKAGE} package", level=3) packages.append(WHEEL_LIBS_PACKAGE) - package_data.update({WHEEL_LIBS_PACKAGE: ['*']}) + package_data.update({WHEEL_LIBS_PACKAGE: ["*"]}) class CopyExt(build_ext): - """Copy extension files to the build directory""" + """Copy extension files to the build directory.""" def run(self): if len(self.extensions) == 1: - self.run_command('build_clib') + self.run_command("build_clib") self.extensions = [] self.extensions = find_prebuilt_extensions(get_dir_list(PY_INSTALL_CFG)) for extension in self.extensions: if not isinstance(extension, PrebuiltExtension): - raise DistutilsSetupError(f'copy_ext can accept PrebuiltExtension only, but got {extension.name}') + raise DistutilsSetupError(f"copy_ext can accept PrebuiltExtension only, but got {extension.name}") src = extension.sources[0] dst = self.get_ext_fullpath(extension.name) os.makedirs(os.path.dirname(dst), exist_ok=True) # setting relative path to find dlls - if sys.platform != 'win32': + if sys.platform != "win32": rpath = os.path.relpath(get_package_dir(PY_INSTALL_CFG), os.path.dirname(src)) - if sys.platform == 'linux': - rpath = os.path.join('$ORIGIN', rpath, WHEEL_LIBS_INSTALL_DIR) - elif sys.platform == 'darwin': - rpath = os.path.join('@loader_path', rpath, WHEEL_LIBS_INSTALL_DIR) + if sys.platform == "linux": + rpath = os.path.join("$ORIGIN", rpath, WHEEL_LIBS_INSTALL_DIR) + elif sys.platform == "darwin": + rpath = os.path.join("@loader_path", rpath, WHEEL_LIBS_INSTALL_DIR) set_rpath(rpath, os.path.realpath(src)) copy_file(src, dst, verbose=self.verbose, dry_run=self.dry_run) class CustomInstall(install): - """Enable build_clib during the installation""" + """Enable build_clib during the installation.""" def run(self): - self.run_command('build') + self.run_command("build") install.run(self) class CustomClean(clean): - """Clean up staging directories""" + """Clean up staging directories.""" def clean(self, install_cfg): for comp, comp_data in install_cfg.items(): - install_prefix = comp_data.get('prefix') - self.announce(f'Cleaning {comp}: {install_prefix}', level=3) + install_prefix = comp_data.get("prefix") + self.announce(f"Cleaning {comp}: {install_prefix}", level=3) if os.path.exists(install_prefix): rmtree(install_prefix) @@ -294,14 +291,12 @@ class CustomClean(clean): def ignore_patterns(*patterns): - """ - Filter names by given patterns - """ + """Filter names by given patterns.""" return lambda name: any(fnmatchcase(name, pat=pat) for pat in patterns) def is_tool(name): - """Check if the command-line tool is available""" + """Check if the command-line tool is available.""" try: devnull = subprocess.DEVNULL subprocess.Popen([name], stdout=devnull, stderr=devnull).communicate() # nosec @@ -312,101 +307,101 @@ def is_tool(name): def remove_rpath(file_path): - """ - Remove rpath from binaries + """Remove rpath from binaries. + :param file_path: binary path :type file_path: pathlib.Path """ - if sys.platform == 'darwin': + if sys.platform == "darwin": cmd = ( - f'otool -l {file_path} ' # noqa: P103 - f'| grep LC_RPATH -A3 ' - f'| grep -o "path.*" ' - f'| cut -d " " -f2 ' - f'| xargs -I{{}} install_name_tool -delete_rpath {{}} {file_path}' + f"otool -l {file_path} " # noqa: P103 + f"| grep LC_RPATH -A3 " + f"| grep -o 'path.*' " + f"| cut -d ' ' -f2 " + f"| xargs -I{{}} install_name_tool -delete_rpath {{}} {file_path}" ) if os.WEXITSTATUS(os.system(cmd)) != 0: # nosec - sys.exit(f'Could not remove rpath for {file_path}') + sys.exit(f"Could not remove rpath for {file_path}") else: - sys.exit(f'Unsupported platform: {sys.platform}') + sys.exit(f"Unsupported platform: {sys.platform}") def set_rpath(rpath, executable): - """Setting rpath for linux and macOS libraries""" - print(f'Setting rpath {rpath} for {executable}') # noqa: T001 + """Setting rpath for linux and macOS libraries.""" + print(f"Setting rpath {rpath} for {executable}") # noqa: T001 cmd = [] - rpath_tool = '' + rpath_tool = "" - if sys.platform == 'linux': - with open(os.path.realpath(executable), 'rb') as file: - if file.read(1) != b'\x7f': - log.warn(f'WARNING: {executable}: missed ELF header') + if sys.platform == "linux": + with open(os.path.realpath(executable), "rb") as file: + if file.read(1) != b"\x7f": + log.warn(f"WARNING: {executable}: missed ELF header") return - rpath_tool = 'patchelf' - cmd = [rpath_tool, '--set-rpath', rpath, executable] - elif sys.platform == 'darwin': - rpath_tool = 'install_name_tool' - cmd = [rpath_tool, '-add_rpath', rpath, executable] + rpath_tool = "patchelf" + cmd = [rpath_tool, "--set-rpath", rpath, executable] + elif sys.platform == "darwin": + rpath_tool = "install_name_tool" + cmd = [rpath_tool, "-add_rpath", rpath, executable] else: - sys.exit(f'Unsupported platform: {sys.platform}') + sys.exit(f"Unsupported platform: {sys.platform}") if is_tool(rpath_tool): - if sys.platform == 'darwin': + if sys.platform == "darwin": remove_rpath(executable) ret_info = subprocess.run(cmd, check=True, shell=False) # nosec if ret_info.returncode != 0: - sys.exit(f'Could not set rpath: {rpath} for {executable}') + sys.exit(f"Could not set rpath: {rpath} for {executable}") else: - sys.exit(f'Could not found {rpath_tool} on the system, ' f'please make sure that this tool is installed') + sys.exit(f"Could not found {rpath_tool} on the system, " f"please make sure that this tool is installed") def find_prebuilt_extensions(search_dirs): - """collect prebuilt python extensions""" + """Collect prebuilt python extensions.""" extensions = [] - ext_pattern = '' - if sys.platform == 'linux': - ext_pattern = '**/*.so' - elif sys.platform == 'win32': - ext_pattern = '**/*.pyd' - elif sys.platform == 'darwin': - ext_pattern = '**/*.so' + ext_pattern = "" + if sys.platform == "linux": + ext_pattern = "**/*.so" + elif sys.platform == "win32": + ext_pattern = "**/*.pyd" + elif sys.platform == "darwin": + ext_pattern = "**/*.so" for base_dir in search_dirs: for path in Path(base_dir).glob(ext_pattern): - if path.match('openvino/libs/*'): + if path.match("openvino/libs/*"): continue relpath = path.relative_to(base_dir) - if relpath.parent != '.': + if relpath.parent != ".": package_names = str(relpath.parent).split(os.path.sep) else: package_names = [] - package_names.append(path.name.split('.', 1)[0]) - name = '.'.join(package_names) + package_names.append(path.name.split(".", 1)[0]) + name = ".".join(package_names) extensions.append(PrebuiltExtension(name, sources=[str(path)])) if not extensions: - extensions.append(PrebuiltExtension('openvino', sources=[str('setup.py')])) + extensions.append(PrebuiltExtension("openvino", sources=[str("setup.py")])) return extensions def get_description(desc_file_path): - """read description from README.md""" - with open(desc_file_path, 'r', encoding='utf-8') as fstream: + """Read description from README.md.""" + with open(desc_file_path, "r", encoding="utf-8") as fstream: description = fstream.read() return description def get_dependencies(requirements_file_path): - """read dependencies from requirements.txt""" - with open(requirements_file_path, 'r', encoding='utf-8') as fstream: + """Read dependencies from requirements.txt.""" + with open(requirements_file_path, "r", encoding="utf-8") as fstream: dependencies = fstream.read() return dependencies def get_dir_list(install_cfg): - """collect all available directories with libs or python packages""" + """Collect all available directories with libs or python packages.""" dirs = [] for comp_info in install_cfg.values(): - cfg_prefix = comp_info.get('prefix') - cfg_dir = comp_info.get('install_dir') + cfg_prefix = comp_info.get("prefix") + cfg_dir = comp_info.get("install_dir") if cfg_dir: if not os.path.isabs(cfg_dir): cfg_dir = os.path.join(cfg_prefix, cfg_dir) @@ -416,11 +411,8 @@ def get_dir_list(install_cfg): def get_package_dir(install_cfg): - """ - Get python package path based on config - All the packages should be located in one directory - """ - py_package_path = '' + """Get python package path based on config. All the packages should be located in one directory.""" + py_package_path = "" dirs = get_dir_list(install_cfg) if len(dirs) != 0: # setup.py support only one package directory, all modules should be located there @@ -429,62 +421,62 @@ def get_package_dir(install_cfg): def concat_files(output_file, input_files): - with open(output_file, 'w', encoding='utf-8') as outfile: + with open(output_file, "w", encoding="utf-8") as outfile: for filename in input_files: - with open(filename, 'r', encoding='utf-8') as infile: + with open(filename, "r", encoding="utf-8") as infile: content = infile.read() outfile.write(content) return output_file -platforms = ['linux', 'win32', 'darwin'] +platforms = ["linux", "win32", "darwin"] if not any(pl in sys.platform for pl in platforms): - sys.exit(f'Unsupported platform: {sys.platform}, expected: linux, win32, darwin') + sys.exit(f"Unsupported platform: {sys.platform}, expected: linux, win32, darwin") # copy license file into the build directory -package_license = os.getenv('WHEEL_LICENSE', SCRIPT_DIR.parents[3] / 'LICENSE') +package_license = os.getenv("WHEEL_LICENSE", SCRIPT_DIR.parents[3] / "LICENSE") if os.path.exists(package_license): - copyfile(package_license, 'LICENSE') + copyfile(package_license, "LICENSE") packages = find_namespace_packages(get_package_dir(PY_INSTALL_CFG)) package_data: typing.Dict[str, list] = {} -pkg_name = os.getenv('WHEEL_PACKAGE_NAME', 'openvino') -ext_modules = find_prebuilt_extensions(get_dir_list(PY_INSTALL_CFG)) if pkg_name == 'openvino' else [] +pkg_name = os.getenv("WHEEL_PACKAGE_NAME", "openvino") +ext_modules = find_prebuilt_extensions(get_dir_list(PY_INSTALL_CFG)) if pkg_name == "openvino" else [] -description_md = SCRIPT_DIR.parents[3] / 'docs' / 'install_guides' / 'pypi-openvino-rt.md' -md_files = [description_md, SCRIPT_DIR.parents[3] / 'docs' / 'install_guides' / 'pre-release-note.md'] -docs_url = 'https://docs.openvino.ai/latest/index.html' +description_md = SCRIPT_DIR.parents[3] / "docs" / "install_guides" / "pypi-openvino-rt.md" +md_files = [description_md, SCRIPT_DIR.parents[3] / "docs" / "install_guides" / "pre-release-note.md"] +docs_url = "https://docs.openvino.ai/latest/index.html" -if(os.getenv('CI_BUILD_DEV_TAG')): - output = Path.cwd() / 'build' / 'pypi-openvino-rt.md' +if(os.getenv("CI_BUILD_DEV_TAG")): + output = Path.cwd() / "build" / "pypi-openvino-rt.md" output.parent.mkdir(exist_ok=True) description_md = concat_files(output, md_files) - docs_url = 'https://docs.openvino.ai/nightly/index.html' + docs_url = "https://docs.openvino.ai/nightly/index.html" setup( - version=os.getenv('WHEEL_VERSION', '0.0.0'), - build=os.getenv('WHEEL_BUILD', '000'), - author_email=os.getenv('WHEEL_AUTHOR_EMAIL', 'openvino_pushbot@intel.com'), + version=os.getenv("WHEEL_VERSION", "0.0.0"), + build=os.getenv("WHEEL_BUILD", "000"), + author_email=os.getenv("WHEEL_AUTHOR_EMAIL", "openvino_pushbot@intel.com"), name=pkg_name, - license=os.getenv('WHEEL_LICENCE_TYPE', 'OSI Approved :: Apache Software License'), - author=os.getenv('WHEEL_AUTHOR', 'Intel(R) Corporation'), - description=os.getenv('WHEEL_DESC', 'OpenVINO(TM) Runtime'), - install_requires=get_dependencies(os.getenv('WHEEL_REQUIREMENTS', SCRIPT_DIR.parents[0] / 'requirements.txt')), - long_description=get_description(os.getenv('WHEEL_OVERVIEW', description_md)), - long_description_content_type='text/markdown', - download_url=os.getenv('WHEEL_DOWNLOAD_URL', 'https://github.com/openvinotoolkit/openvino/tags'), - url=os.getenv('WHEEL_URL', docs_url), + license=os.getenv("WHEEL_LICENCE_TYPE", "OSI Approved :: Apache Software License"), + author=os.getenv("WHEEL_AUTHOR", "Intel(R) Corporation"), + description=os.getenv("WHEEL_DESC", "OpenVINO(TM) Runtime"), + install_requires=get_dependencies(os.getenv("WHEEL_REQUIREMENTS", SCRIPT_DIR.parents[0] / "requirements.txt")), + long_description=get_description(os.getenv("WHEEL_OVERVIEW", description_md)), + long_description_content_type="text/markdown", + download_url=os.getenv("WHEEL_DOWNLOAD_URL", "https://github.com/openvinotoolkit/openvino/tags"), + url=os.getenv("WHEEL_URL", docs_url), cmdclass={ - 'build': CustomBuild, - 'install': CustomInstall, - 'build_clib': PrepareLibs, - 'build_ext': CopyExt, - 'clean': CustomClean, + "build": CustomBuild, + "install": CustomInstall, + "build_clib": PrepareLibs, + "build_ext": CopyExt, + "clean": CustomClean, }, ext_modules=ext_modules, packages=packages, - package_dir={'': get_package_dir(PY_INSTALL_CFG)}, + package_dir={"": get_package_dir(PY_INSTALL_CFG)}, package_data=package_data, zip_safe=False, ) From d107cec39fa0f0ad774e1bcd8ce93dce311dd220 Mon Sep 17 00:00:00 2001 From: Anastasia Kuporosova Date: Thu, 31 Mar 2022 00:33:57 +0300 Subject: [PATCH 12/39] [Python API] Update names in Model class (#11348) * [Python API] Update names in Model class * fix code-style * remove from_capsule in the new API --- .../python/src/pyopenvino/graph/model.cpp | 494 ++++++++---------- .../src/pyopenvino/graph/partial_shape.cpp | 16 - 2 files changed, 227 insertions(+), 283 deletions(-) diff --git a/src/bindings/python/src/pyopenvino/graph/model.cpp b/src/bindings/python/src/pyopenvino/graph/model.cpp index 39af25c87fd..5bc4b6e4dcc 100644 --- a/src/bindings/python/src/pyopenvino/graph/model.cpp +++ b/src/bindings/python/src/pyopenvino/graph/model.cpp @@ -19,8 +19,6 @@ namespace py = pybind11; -static const char* CAPSULE_NAME = "openvino_function"; - using PyRTMap = ov::RTMap; PYBIND11_MAKE_OPAQUE(PyRTMap); @@ -46,22 +44,22 @@ ov::SinkVector cast_to_sink_vector(const std::vector>& } void regclass_graph_Model(py::module m) { - py::class_> function(m, "Model", py::module_local()); - function.doc() = "openvino.runtime.Model wraps ov::Model"; + py::class_> model(m, "Model", py::module_local()); + model.doc() = "openvino.runtime.Model wraps ov::Model"; - function.def(py::init([](const ov::ResultVector& res, - const std::vector>& nodes, - const ov::ParameterVector& params, - const std::string& name) { - set_tensor_names(params); - const auto sinks = cast_to_sink_vector(nodes); - return std::make_shared(res, sinks, params, name); - }), - py::arg("results"), - py::arg("sinks"), - py::arg("parameters"), - py::arg("name"), - R"( + model.def(py::init([](const ov::ResultVector& res, + const std::vector>& nodes, + const ov::ParameterVector& params, + const std::string& name) { + set_tensor_names(params); + const auto sinks = cast_to_sink_vector(nodes); + return std::make_shared(res, sinks, params, name); + }), + py::arg("results"), + py::arg("sinks"), + py::arg("parameters"), + py::arg("name"), + R"( Create user-defined Model which is a representation of a model. :param results: List of results. @@ -70,51 +68,51 @@ void regclass_graph_Model(py::module m) { :type sinks: List[openvino.runtime.Node] :param parameters: List of parameters. :type parameters: List[op.Parameter] - :param name: String to set as function's friendly name. + :param name: String to set as model's friendly name. :type name: str )"); - function.def(py::init([](const std::vector>& results, - const ov::ParameterVector& parameters, - const std::string& name) { - set_tensor_names(parameters); - return std::make_shared(results, parameters, name); - }), - py::arg("results"), - py::arg("parameters"), - py::arg("name") = "", - R"( + model.def(py::init([](const std::vector>& results, + const ov::ParameterVector& parameters, + const std::string& name) { + set_tensor_names(parameters); + return std::make_shared(results, parameters, name); + }), + py::arg("results"), + py::arg("parameters"), + py::arg("name") = "", + R"( Create user-defined Model which is a representation of a model. :param results: List of Nodes to be used as results. :type results: List[openvino.runtime.Node] :param parameters: List of parameters. :type parameters: List[op.Parameter] - :param name: String to set as function's friendly name. + :param name: String to set as model's friendly name. :type name: str )"); - function.def(py::init([](const std::shared_ptr& result, - const ov::ParameterVector& parameters, - const std::string& name) { - set_tensor_names(parameters); - return std::make_shared(result, parameters, name); - }), - py::arg("result"), - py::arg("parameters"), - py::arg("name") = "", - R"( + model.def(py::init([](const std::shared_ptr& result, + const ov::ParameterVector& parameters, + const std::string& name) { + set_tensor_names(parameters); + return std::make_shared(result, parameters, name); + }), + py::arg("result"), + py::arg("parameters"), + py::arg("name") = "", + R"( Create user-defined Model which is a representation of a model. :param result: Node to be used as result. :type result: openvino.runtime.Node :param parameters: List of parameters. :type parameters: List[op.Parameter] - :param name: String to set as function's friendly name. + :param name: String to set as model's friendly name. :type name: str )"); - function.def( + model.def( py::init([](const ov::OutputVector& results, const ov::ParameterVector& parameters, const std::string& name) { set_tensor_names(parameters); return std::make_shared(results, parameters, name); @@ -129,47 +127,47 @@ void regclass_graph_Model(py::module m) { :type results: List[openvino.runtime.Output] :param parameters: List of parameters. :type parameters: List[op.Parameter] - :param name: String to set as function's friendly name. + :param name: String to set as model's friendly name. :type name: str )"); - function.def(py::init([](const ov::OutputVector& results, - const std::vector>& nodes, - const ov::ParameterVector& parameters, - const std::string& name) { - set_tensor_names(parameters); - const auto sinks = cast_to_sink_vector(nodes); - return std::make_shared(results, sinks, parameters, name); - }), - py::arg("results"), - py::arg("sinks"), - py::arg("parameters"), - py::arg("name") = "", - R"( + model.def(py::init([](const ov::OutputVector& results, + const std::vector>& nodes, + const ov::ParameterVector& parameters, + const std::string& name) { + set_tensor_names(parameters); + const auto sinks = cast_to_sink_vector(nodes); + return std::make_shared(results, sinks, parameters, name); + }), + py::arg("results"), + py::arg("sinks"), + py::arg("parameters"), + py::arg("name") = "", + R"( Create user-defined Model which is a representation of a model :param results: List of outputs. :type results: List[openvino.runtime.Output] :param sinks: List of Nodes to be used as Sinks (e.g. Assign ops). :type sinks: List[openvino.runtime.Node] - :param name: String to set as function's friendly name. + :param name: String to set as model's friendly name. :type name: str )"); - function.def(py::init([](const ov::ResultVector& results, - const std::vector>& nodes, - const ov::ParameterVector& parameters, - const ov::op::util::VariableVector& variables, - const std::string& name) { - set_tensor_names(parameters); - const auto sinks = cast_to_sink_vector(nodes); - return std::make_shared(results, sinks, parameters, variables, name); - }), - py::arg("results"), - py::arg("sinks"), - py::arg("parameters"), - py::arg("variables"), - py::arg("name") = "", - R"( + model.def(py::init([](const ov::ResultVector& results, + const std::vector>& nodes, + const ov::ParameterVector& parameters, + const ov::op::util::VariableVector& variables, + const std::string& name) { + set_tensor_names(parameters); + const auto sinks = cast_to_sink_vector(nodes); + return std::make_shared(results, sinks, parameters, variables, name); + }), + py::arg("results"), + py::arg("sinks"), + py::arg("parameters"), + py::arg("variables"), + py::arg("name") = "", + R"( Create user-defined Model which is a representation of a model :param results: List of results. @@ -180,25 +178,25 @@ void regclass_graph_Model(py::module m) { :type parameters: List[op.Parameter] :param variables: List of variables. :type variables: List[op.util.Variable] - :param name: String to set as function's friendly name. + :param name: String to set as model's friendly name. :type name: str )"); - function.def(py::init([](const ov::OutputVector& results, - const std::vector>& nodes, - const ov::ParameterVector& parameters, - const ov::op::util::VariableVector& variables, - const std::string& name) { - set_tensor_names(parameters); - const auto sinks = cast_to_sink_vector(nodes); - return std::make_shared(results, sinks, parameters, variables, name); - }), - py::arg("results"), - py::arg("sinks"), - py::arg("parameters"), - py::arg("variables"), - py::arg("name") = "", - R"( + model.def(py::init([](const ov::OutputVector& results, + const std::vector>& nodes, + const ov::ParameterVector& parameters, + const ov::op::util::VariableVector& variables, + const std::string& name) { + set_tensor_names(parameters); + const auto sinks = cast_to_sink_vector(nodes); + return std::make_shared(results, sinks, parameters, variables, name); + }), + py::arg("results"), + py::arg("sinks"), + py::arg("parameters"), + py::arg("variables"), + py::arg("name") = "", + R"( Create user-defined Model which is a representation of a model :param results: List of results. @@ -207,22 +205,22 @@ void regclass_graph_Model(py::module m) { :type sinks: List[openvino.runtime.Node] :param variables: List of variables. :type variables: List[op.util.Variable] - :param name: String to set as function's friendly name. + :param name: String to set as model's friendly name. :type name: str )"); - function.def(py::init([](const ov::ResultVector& results, - const ov::ParameterVector& parameters, - const ov::op::util::VariableVector& variables, - const std::string& name) { - set_tensor_names(parameters); - return std::make_shared(results, parameters, variables, name); - }), - py::arg("results"), - py::arg("parameters"), - py::arg("variables"), - py::arg("name") = "", - R"( + model.def(py::init([](const ov::ResultVector& results, + const ov::ParameterVector& parameters, + const ov::op::util::VariableVector& variables, + const std::string& name) { + set_tensor_names(parameters); + return std::make_shared(results, parameters, variables, name); + }), + py::arg("results"), + py::arg("parameters"), + py::arg("variables"), + py::arg("name") = "", + R"( Create user-defined Model which is a representation of a model :param results: List of results. @@ -231,35 +229,35 @@ void regclass_graph_Model(py::module m) { :type parameters: List[op.Parameter] :param variables: List of variables. :type variables: List[op.util.Variable] - :param name: String to set as function's friendly name. + :param name: String to set as model's friendly name. :type name: str )"); - function.def(py::init([](const ov::OutputVector& results, - const ov::ParameterVector& parameters, - const ov::op::util::VariableVector& variables, - const std::string& name) { - set_tensor_names(parameters); - return std::make_shared(results, parameters, variables, name); - }), - py::arg("results"), - py::arg("parameters"), - py::arg("variables"), - py::arg("name") = "", - R"( + model.def(py::init([](const ov::OutputVector& results, + const ov::ParameterVector& parameters, + const ov::op::util::VariableVector& variables, + const std::string& name) { + set_tensor_names(parameters); + return std::make_shared(results, parameters, variables, name); + }), + py::arg("results"), + py::arg("parameters"), + py::arg("variables"), + py::arg("name") = "", + R"( Create user-defined Model which is a representation of a model :param results: List of results. :type results: List[openvino.runtime.Output] :param parameters: List of parameters. :type parameters: List[op.Parameter] - :param name: String to set as function's friendly name. + :param name: String to set as model's friendly name. :type name: str )"); - function.def("validate_nodes_and_infer_types", &ov::Model::validate_nodes_and_infer_types); + model.def("validate_nodes_and_infer_types", &ov::Model::validate_nodes_and_infer_types); - function.def( + model.def( "reshape", [](ov::Model& self, const ov::PartialShape& partial_shape) { self.reshape(partial_shape); @@ -271,7 +269,7 @@ void regclass_graph_Model(py::module m) { :return : void )"); - function.def( + model.def( "reshape", [](ov::Model& self, const py::list& partial_shape) { self.reshape(Common::partial_shape_from_list(partial_shape)); @@ -283,7 +281,7 @@ void regclass_graph_Model(py::module m) { :return : void )"); - function.def( + model.def( "reshape", [](ov::Model& self, const py::tuple& partial_shape) { self.reshape(Common::partial_shape_from_list(partial_shape.cast())); @@ -295,7 +293,7 @@ void regclass_graph_Model(py::module m) { :return : void )"); - function.def( + model.def( "reshape", [](ov::Model& self, const std::string& partial_shape) { self.reshape(Common::partial_shape_from_str(partial_shape)); @@ -307,7 +305,7 @@ void regclass_graph_Model(py::module m) { :return : void )"); - function.def( + model.def( "reshape", [](ov::Model& self, const py::dict& partial_shapes) { std::map, ov::PartialShape> new_shapes; @@ -373,34 +371,34 @@ void regclass_graph_Model(py::module m) { :type partial_shapes: Dict[keys, values] )"); - function.def("get_output_size", - &ov::Model::get_output_size, - R"( - Return the number of outputs for the function. + model.def("get_output_size", + &ov::Model::get_output_size, + R"( + Return the number of outputs for the model. :return: Number of outputs. :rtype: int )"); - function.def("get_ops", - &ov::Model::get_ops, - R"( - Return ops used in the function. + model.def("get_ops", + &ov::Model::get_ops, + R"( + Return ops used in the model. - :return: List of Nodes representing ops used in function. + :return: List of Nodes representing ops used in model. :rtype: List[openvino.runtime.Node] )"); - function.def("get_ordered_ops", - &ov::Model::get_ordered_ops, - R"( - Return ops used in the function in topological order. + model.def("get_ordered_ops", + &ov::Model::get_ordered_ops, + R"( + Return ops used in the model in topological order. - :return: List of sorted Nodes representing ops used in function. + :return: List of sorted Nodes representing ops used in model. :rtype: List[openvino.runtime.Node] )"); - function.def("get_output_op", - &ov::Model::get_output_op, - py::arg("index"), - R"( + model.def("get_output_op", + &ov::Model::get_output_op, + py::arg("index"), + R"( Return the op that generates output i :param index: output index @@ -408,10 +406,10 @@ void regclass_graph_Model(py::module m) { :return: Node object that generates output i :rtype: openvino.runtime.Node )"); - function.def("get_output_element_type", - &ov::Model::get_output_element_type, - py::arg("index"), - R"( + model.def("get_output_element_type", + &ov::Model::get_output_element_type, + py::arg("index"), + R"( Return the element type of output i :param index: output index @@ -419,10 +417,10 @@ void regclass_graph_Model(py::module m) { :return: Type object of output i :rtype: openvino.runtime.Type )"); - function.def("get_output_shape", - &ov::Model::get_output_shape, - py::arg("index"), - R"( + model.def("get_output_shape", + &ov::Model::get_output_shape, + py::arg("index"), + R"( Return the shape of element i :param index: element index @@ -430,10 +428,10 @@ void regclass_graph_Model(py::module m) { :return: Shape object of element i :rtype: openvino.runtime.Shape )"); - function.def("get_output_partial_shape", - &ov::Model::get_output_partial_shape, - py::arg("index"), - R"( + model.def("get_output_partial_shape", + &ov::Model::get_output_partial_shape, + py::arg("index"), + R"( Return the partial shape of element i :param index: element index @@ -441,34 +439,34 @@ void regclass_graph_Model(py::module m) { :return: PartialShape object of element i :rtype: openvino.runtime.PartialShape )"); - function.def("get_parameters", - &ov::Model::get_parameters, - R"( - Return the function parameters. + model.def("get_parameters", + &ov::Model::get_parameters, + R"( + Return the model parameters. - :return: ParameterVector containing function parameters. + :return: ParameterVector containing model parameters. :rtype: ParameterVector )"); - function.def("get_results", - &ov::Model::get_results, - R"( - Return a list of function outputs. + model.def("get_results", + &ov::Model::get_results, + R"( + Return a list of model outputs. - :return: ResultVector containing function parameters. + :return: ResultVector containing model parameters. :rtype: ResultVector )"); - function.def("get_result", - &ov::Model::get_result, - R"( + model.def("get_result", + &ov::Model::get_result, + R"( Return single result. :return: Node object representing result. :rtype: openvino.runtime.Node )"); - function.def("get_result_index", - (int64_t(ov::Model::*)(const ov::Output&) const) & ov::Model::get_result_index, - py::arg("value"), - R"( + model.def("get_result_index", + (int64_t(ov::Model::*)(const ov::Output&) const) & ov::Model::get_result_index, + py::arg("value"), + R"( Return index of result. Return -1 if `value` not matched. @@ -478,10 +476,10 @@ void regclass_graph_Model(py::module m) { :return: Index for value referencing it. :rtype: int )"); - function.def("get_result_index", - (int64_t(ov::Model::*)(const ov::Output&) const) & ov::Model::get_result_index, - py::arg("value"), - R"( + model.def("get_result_index", + (int64_t(ov::Model::*)(const ov::Output&) const) & ov::Model::get_result_index, + py::arg("value"), + R"( Return index of result. Return -1 if `value` not matched. @@ -492,81 +490,77 @@ void regclass_graph_Model(py::module m) { :rtype: int )"); - function.def("get_name", - &ov::Model::get_name, - R"( - Get the unique name of the function. + model.def("get_name", + &ov::Model::get_name, + R"( + Get the unique name of the model. - :return: String with a name of the function. + :return: String with a name of the model. :rtype: str )"); - function.def("get_friendly_name", - &ov::Model::get_friendly_name, - R"( - Gets the friendly name for a function. If no + model.def("get_friendly_name", + &ov::Model::get_friendly_name, + R"( + Gets the friendly name for a model. If no friendly name has been set via set_friendly_name - then the function's unique name is returned. + then the model's unique name is returned. - :return: String with a friendly name of the function. + :return: String with a friendly name of the model. :rtype: str )"); - function.def("set_friendly_name", - &ov::Model::set_friendly_name, - py::arg("name"), - R"( - Sets a friendly name for a function. This does - not overwrite the unique name of the function and + model.def("set_friendly_name", + &ov::Model::set_friendly_name, + py::arg("name"), + R"( + Sets a friendly name for a model. This does + not overwrite the unique name of the model and is retrieved via get_friendly_name(). Used mainly for debugging. :param name: String to set as the friendly name. :type name: str )"); - function.def("is_dynamic", - &ov::Model::is_dynamic, - R"( - Returns true if any of the op's defined in the function + model.def("is_dynamic", + &ov::Model::is_dynamic, + R"( + Returns true if any of the op's defined in the model contains partial shape. :rtype: bool )"); - function.def("input", (ov::Output(ov::Model::*)()) & ov::Model::input); + model.def("input", (ov::Output(ov::Model::*)()) & ov::Model::input); - function.def("input", (ov::Output(ov::Model::*)(size_t)) & ov::Model::input, py::arg("index")); + model.def("input", (ov::Output(ov::Model::*)(size_t)) & ov::Model::input, py::arg("index")); - function.def("input", - (ov::Output(ov::Model::*)(const std::string&)) & ov::Model::input, - py::arg("tensor_name")); + model.def("input", + (ov::Output(ov::Model::*)(const std::string&)) & ov::Model::input, + py::arg("tensor_name")); - function.def("input", (ov::Output(ov::Model::*)() const) & ov::Model::input); + model.def("input", (ov::Output(ov::Model::*)() const) & ov::Model::input); - function.def("input", - (ov::Output(ov::Model::*)(size_t) const) & ov::Model::input, - py::arg("index")); + model.def("input", (ov::Output(ov::Model::*)(size_t) const) & ov::Model::input, py::arg("index")); - function.def("input", - (ov::Output(ov::Model::*)(const std::string&) const) & ov::Model::input, - py::arg("tensor_name")); + model.def("input", + (ov::Output(ov::Model::*)(const std::string&) const) & ov::Model::input, + py::arg("tensor_name")); - function.def("output", (ov::Output(ov::Model::*)()) & ov::Model::output); + model.def("output", (ov::Output(ov::Model::*)()) & ov::Model::output); - function.def("output", (ov::Output(ov::Model::*)(size_t)) & ov::Model::output, py::arg("index")); + model.def("output", (ov::Output(ov::Model::*)(size_t)) & ov::Model::output, py::arg("index")); - function.def("output", - (ov::Output(ov::Model::*)(const std::string&)) & ov::Model::output, - py::arg("tensor_name")); + model.def("output", + (ov::Output(ov::Model::*)(const std::string&)) & ov::Model::output, + py::arg("tensor_name")); - function.def("output", (ov::Output(ov::Model::*)() const) & ov::Model::output); + model.def("output", (ov::Output(ov::Model::*)() const) & ov::Model::output); - function.def("output", - (ov::Output(ov::Model::*)(size_t) const) & ov::Model::output, - py::arg("index")); + model.def("output", (ov::Output(ov::Model::*)(size_t) const) & ov::Model::output, py::arg("index")); - function.def("output", - (ov::Output(ov::Model::*)(const std::string&) const) & ov::Model::output, - py::arg("tensor_name")); + model.def("output", + (ov::Output(ov::Model::*)(const std::string&) const) & ov::Model::output, + py::arg("tensor_name")); - function.def( + model.def( "add_outputs", [](ov::Model& self, py::handle& outputs) { int i = 0; @@ -606,15 +600,15 @@ void regclass_graph_Model(py::module m) { }, py::arg("outputs")); - function.def("replace_parameter", - &ov::Model::replace_parameter, - py::arg("parameter_index"), - py::arg("parameter"), - R"( - Replace the `parameter_index` parameter of the function with `parameter` + model.def("replace_parameter", + &ov::Model::replace_parameter, + py::arg("parameter_index"), + py::arg("parameter"), + R"( + Replace the `parameter_index` parameter of the model with `parameter` All users of the `parameter_index` parameter are redirected to `parameter` , and the - `parameter_index` entry in the function parameter list is replaced with `parameter` + `parameter_index` entry in the model parameter list is replaced with `parameter` :param parameter_index: The index of the parameter to replace. :type parameter_index: int @@ -622,7 +616,7 @@ void regclass_graph_Model(py::module m) { :type parameter: op.Parameter )"); - function.def( + model.def( "get_parameter_index", (int64_t(ov::Model::*)(const std::shared_ptr&) const) & ov::Model::get_parameter_index, py::arg("parameter"), @@ -637,7 +631,7 @@ void regclass_graph_Model(py::module m) { :rtype: int )"); - function.def( + model.def( "evaluate", [](ov::Model& self, ov::TensorVector& output_tensors, @@ -649,20 +643,20 @@ void regclass_graph_Model(py::module m) { py::arg("input_tensors"), py::arg("evaluation_context") = PyRTMap(), R"( - Evaluate the function on inputs, putting results in outputs + Evaluate the model on inputs, putting results in outputs :param output_tensors: Tensors for the outputs to compute. One for each result :type output_tensors: List[openvino.runtime.Tensor] :param input_tensors: Tensors for the inputs. One for each inputs. :type input_tensors: List[openvino.runtime.Tensor] :param evaluation_context: Storage of additional settings and attributes that can be used - when evaluating the function. This additional information can be + when evaluating the model. This additional information can be shared across nodes. :type evaluation_context: openvino.runtime.RTMap :rtype: bool )"); - function.def("__repr__", [](const ov::Model& self) { + model.def("__repr__", [](const ov::Model& self) { std::string class_name = py::cast(self).get_type().attr("__name__").cast(); auto inputs_str = Common::docs::container_to_string(self.inputs(), ",\n"); @@ -672,42 +666,8 @@ void regclass_graph_Model(py::module m) { outputs_str + "\n]>"; }); - function.def_static("from_capsule", [](py::object* capsule) { - // get the underlying PyObject* which is a PyCapsule pointer - auto* pybind_capsule_ptr = capsule->ptr(); - // extract the pointer stored in the PyCapsule under the name CAPSULE_NAME - auto* capsule_ptr = PyCapsule_GetPointer(pybind_capsule_ptr, CAPSULE_NAME); - - auto* ngraph_function = static_cast*>(capsule_ptr); - if (ngraph_function && *ngraph_function) { - return *ngraph_function; - } else { - throw std::runtime_error("The provided capsule does not contain an ov::Model"); - } - }); - function.def_static("to_capsule", [](std::shared_ptr& ngraph_function) { - // create a shared pointer on the heap before putting it in the capsule - // this secures the lifetime of the object transferred by the capsule - auto* sp_copy = new std::shared_ptr(ngraph_function); - - // a destructor callback that will delete the heap allocated shared_ptr - // when the capsule is destructed - auto sp_deleter = [](PyObject* capsule) { - auto* capsule_ptr = PyCapsule_GetPointer(capsule, CAPSULE_NAME); - auto* function_sp = static_cast*>(capsule_ptr); - if (function_sp) { - delete function_sp; - } - }; - - // put the shared_ptr in a new capsule under the same name as in "from_capsule" - auto pybind_capsule = py::capsule(sp_copy, CAPSULE_NAME, sp_deleter); - - return pybind_capsule; - }); - - function.def_property_readonly("inputs", (std::vector>(ov::Model::*)()) & ov::Model::inputs); - function.def_property_readonly("outputs", (std::vector>(ov::Model::*)()) & ov::Model::outputs); - function.def_property_readonly("name", &ov::Model::get_name); - function.def_property("friendly_name", &ov::Model::get_friendly_name, &ov::Model::set_friendly_name); + model.def_property_readonly("inputs", (std::vector>(ov::Model::*)()) & ov::Model::inputs); + model.def_property_readonly("outputs", (std::vector>(ov::Model::*)()) & ov::Model::outputs); + model.def_property_readonly("name", &ov::Model::get_name); + model.def_property("friendly_name", &ov::Model::get_friendly_name, &ov::Model::set_friendly_name); } diff --git a/src/bindings/python/src/pyopenvino/graph/partial_shape.cpp b/src/bindings/python/src/pyopenvino/graph/partial_shape.cpp index dfc5595f7fe..82aaa62a974 100644 --- a/src/bindings/python/src/pyopenvino/graph/partial_shape.cpp +++ b/src/bindings/python/src/pyopenvino/graph/partial_shape.cpp @@ -18,8 +18,6 @@ namespace py = pybind11; -static const char* CAPSULE_NAME = "ngraph_partial_shape"; - void regclass_graph_PartialShape(py::module m) { py::class_> shape(m, "PartialShape"); shape.doc() = "openvino.runtime.PartialShape wraps ov::PartialShape"; @@ -193,18 +191,4 @@ void regclass_graph_PartialShape(py::module m) { shape.def("__repr__", [](const ov::PartialShape& self) -> std::string { return "() + ">"; }); - - shape.def_static("from_capsule", [](py::object* capsule) { - // get the underlying PyObject* which is a PyCapsule pointer - auto* pybind_capsule_ptr = capsule->ptr(); - // extract the pointer stored in the PyCapsule under the name CAPSULE_NAME - auto* capsule_ptr = PyCapsule_GetPointer(pybind_capsule_ptr, CAPSULE_NAME); - - auto* ngraph_pShape = static_cast*>(capsule_ptr); - if (ngraph_pShape && *ngraph_pShape) { - return *ngraph_pShape; - } else { - throw std::runtime_error("The provided capsule does not contain an ov::PartialShape"); - } - }); } From 88e20199f02b00f422eddf5eb766d59f0aae1bfb Mon Sep 17 00:00:00 2001 From: Oleg Pipikin Date: Thu, 31 Mar 2022 07:24:46 +0300 Subject: [PATCH 13/39] Fix query network for hetero plugin (#10556) * Fix query network for hetero plugin * Apply comments * Fix1 * Add tests * Apply comments 2 * Apply comments 3 --- .../interface/ie_iplugin_internal.hpp | 11 + .../interface/ie_iplugin_internal.cpp | 21 + src/plugins/intel_cpu/src/plugin.cpp | 45 +- src/plugins/intel_gpu/src/plugin/plugin.cpp | 61 ++- .../common/src/ngraph/query_network.cpp | 19 +- .../myriad_plugin/myriad_plugin.cpp | 7 + .../cpu/behavior/hetero_cpu_query_network.cpp | 22 + .../gpu/behavior/hetero_gpu_query_network.cpp | 22 + .../behavior/plugin/hetero_query_network.hpp | 466 ++++++++++++++++++ 9 files changed, 645 insertions(+), 29 deletions(-) create mode 100644 src/tests/functional/plugin/cpu/behavior/hetero_cpu_query_network.cpp create mode 100644 src/tests/functional/plugin/gpu/behavior/hetero_gpu_query_network.cpp create mode 100644 src/tests/functional/plugin/shared/include/behavior/plugin/hetero_query_network.hpp diff --git a/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp b/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp index e164a65c70d..d476724fb68 100644 --- a/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp +++ b/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp @@ -335,6 +335,17 @@ protected: void SetExeNetworkInfo(const std::shared_ptr& exeNetwork, const std::shared_ptr& function); + /** + * @brief Returns set of nodes which were removed after transformation. + * If originalFunction contains node1 and transformedFunction does not + * contains node1 in ops list, node1 will be returned. + * @param originalFunction Original network + * @param transformedFunction Transformed network + * @return Set of strings which contains removed node names + */ + std::unordered_set GetRemovedNodes(const std::shared_ptr& originalFunction, + const std::shared_ptr& transformedFunction) const; + std::string _pluginName; //!< A device name that plugins enables std::map _config; //!< A map config keys -> values std::weak_ptr _core; //!< A pointer to ICore interface diff --git a/src/inference/src/cpp_interfaces/interface/ie_iplugin_internal.cpp b/src/inference/src/cpp_interfaces/interface/ie_iplugin_internal.cpp index 92ac8e919f6..8fbccabb034 100644 --- a/src/inference/src/cpp_interfaces/interface/ie_iplugin_internal.cpp +++ b/src/inference/src/cpp_interfaces/interface/ie_iplugin_internal.cpp @@ -20,6 +20,7 @@ #include "cnn_network_ngraph_impl.hpp" #include "cpp/ie_cnn_network.h" #include "exec_graph_info.hpp" +#include "ie_algorithm.hpp" #include "ie_api.h" #include "ie_icore.hpp" #include "ie_iextension.h" @@ -300,6 +301,26 @@ void IInferencePlugin::SetExeNetworkInfo(const std::shared_ptrSetPointerToPlugin(shared_from_this()); } +std::unordered_set IInferencePlugin::GetRemovedNodes( + const std::shared_ptr& originalFunction, + const std::shared_ptr& transformedFunction) const { + std::unordered_set result = {}; + std::unordered_set transformedNodeNames = {}; + + for (auto&& node : transformedFunction->get_ops()) { + transformedNodeNames.emplace(node->get_friendly_name()); + for (auto&& fusedLayerName : ngraph::getFusedNamesVector(node)) + transformedNodeNames.emplace(fusedLayerName); + } + + for (auto&& originalNode : originalFunction->get_ops()) { + if (!InferenceEngine::details::contains(transformedNodeNames, originalNode->get_friendly_name())) + result.emplace(originalNode->get_friendly_name()); + } + + return result; +} + void SetExeNetworkInfo(const std::shared_ptr& exeNetwork, const std::shared_ptr& function, bool new_api) { diff --git a/src/plugins/intel_cpu/src/plugin.cpp b/src/plugins/intel_cpu/src/plugin.cpp index d9369bc56b3..d5ab9fafc64 100644 --- a/src/plugins/intel_cpu/src/plugin.cpp +++ b/src/plugins/intel_cpu/src/plugin.cpp @@ -915,28 +915,49 @@ QueryNetworkResult Engine::QueryNetwork(const CNNNetwork& network, const std::ma } auto clonedNetwork = InferenceEngine::details::cloneNetwork(network); + auto clonnedFunction = clonedNetwork.getFunction(); const auto& lptProp = config.find(InferenceEngine::PluginConfigInternalParams::KEY_LP_TRANSFORMS_MODE); const bool enableLPT = (lptProp != config.end() && lptProp->second == PluginConfigParams::YES) /* enabled in the orig_config*/ || Config::LPTransformsMode::On == engConfig.lpTransformsMode /* or already enabled */; const bool enableSnippets = !(conf.cache_dir.empty() || conf.enableDynamicBatch || (conf.enforceBF16 && dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_core))); Transformation(clonedNetwork, enableLPT, enableSnippets, isLegacyAPI()); - auto ops = clonedNetwork.getFunction()->get_ordered_ops(); - std::unordered_set supported; + auto ops = clonnedFunction->get_ordered_ops(); + + //Mark removed nodes as supported + std::unordered_set supported = GetRemovedNodes(function, clonnedFunction);; std::unordered_set unsupported; - for (auto op : ops) { - auto layerIsSupported = [&] { - std::unique_ptr ptr; - try { - ptr.reset(Node::factory().create(op, {mkldnn::engine::kind::cpu, 0}, extensionManager, fake_w_cache)); - } catch (InferenceEngine::Exception&) { - return false; + + auto layerIsSupported = [&](const std::shared_ptr& op) { + std::unique_ptr ptr; + try { + ptr.reset(Node::factory().create(op, {mkldnn::engine::kind::cpu, 0}, extensionManager, fake_w_cache)); + } catch (const InferenceEngine::Exception&) { + return false; + } + return true; + }; + + for (auto&& op : ops) { + bool isSupported = false; + bool wasNodeAlreadyChecked = false; + if (InferenceEngine::details::contains(originalOps, op->get_friendly_name())) { + isSupported = layerIsSupported(op); + wasNodeAlreadyChecked = true; + if (isSupported) { + supported.emplace(op->get_friendly_name()); + } else { + unsupported.emplace(op->get_friendly_name()); } - return true; - } (); + } + for (auto&& fusedLayerName : ngraph::getFusedNamesVector(op)) { if (InferenceEngine::details::contains(originalOps, fusedLayerName)) { - if (layerIsSupported) { + if (!wasNodeAlreadyChecked) { + isSupported = layerIsSupported(op); + wasNodeAlreadyChecked = true; + } + if (isSupported) { supported.emplace(fusedLayerName); } else { unsupported.emplace(fusedLayerName); diff --git a/src/plugins/intel_gpu/src/plugin/plugin.cpp b/src/plugins/intel_gpu/src/plugin/plugin.cpp index 44eb01608ee..cc87261c3e0 100644 --- a/src/plugins/intel_gpu/src/plugin/plugin.cpp +++ b/src/plugins/intel_gpu/src/plugin/plugin.cpp @@ -398,10 +398,14 @@ QueryNetworkResult Plugin::QueryNetwork(const CNNNetwork& network, auto clonedNetwork = CloneAndTransformNetwork(network, conf); auto func = clonedNetwork.getFunction(); auto ops = func->get_ordered_ops(); - std::unordered_set supported; + + //Mark removed nodes as supported + std::unordered_set supported = GetRemovedNodes(function, func);; std::unordered_set unsupported; - std::unordered_set constantsNames; + std::unordered_set supportedNotOriginal; + std::unordered_set unsupportedNotOriginal; + std::vector> constants; std::map shapes; @@ -442,34 +446,46 @@ QueryNetworkResult Plugin::QueryNetwork(const CNNNetwork& network, return false; } if (ngraph::is_type(node)) { - constantsNames.emplace(node->get_friendly_name()); constants.push_back(node); return false; } - return prog.IsOpSupported(network, node) && - !ngraph::op::is_parameter(node) && - !ngraph::op::is_output(node); + return prog.IsOpSupported(network, node) || + ngraph::op::is_parameter(node) || + ngraph::op::is_output(node); }; // Get ops after transformations and check if it's supported // Transformations might lead to the situation when single node is merged to multiple operations, // so we mark original op as supported only if all nodes that it was merged into are supported - bool wasNodeAlreadyChecked = false; - bool isSupported = false; for (auto&& op : ops) { - wasNodeAlreadyChecked = false; - isSupported = false; + bool isSupported = layerIsSupported(op); + if (InferenceEngine::details::contains(originalOpNames, op->get_friendly_name())) { + if (isSupported) { + supported.emplace(op->get_friendly_name()); + } else { + unsupported.emplace(op->get_friendly_name()); + } + } else { + if (isSupported) { + supportedNotOriginal.emplace(op->get_friendly_name()); + } else { + unsupportedNotOriginal.emplace(op->get_friendly_name()); + } + } + for (auto&& fusedLayerName : ngraph::getFusedNamesVector(op)) { if (InferenceEngine::details::contains(originalOpNames, fusedLayerName)) { - if (!wasNodeAlreadyChecked) { - isSupported = layerIsSupported(op); - wasNodeAlreadyChecked = true; - } if (isSupported) { supported.emplace(fusedLayerName); } else { unsupported.emplace(fusedLayerName); } + } else { + if (isSupported) { + supportedNotOriginal.emplace(fusedLayerName); + } else { + unsupportedNotOriginal.emplace(fusedLayerName); + } } } } @@ -481,22 +497,35 @@ QueryNetworkResult Plugin::QueryNetwork(const CNNNetwork& network, } unsupported.clear(); + for (auto&& layerName : unsupportedNotOriginal) { + if (InferenceEngine::details::contains(supportedNotOriginal, layerName)) { + supportedNotOriginal.erase(layerName); + } + } + unsupportedNotOriginal.clear(); + // 1. Constants are marked as supported when all outputs can be offloaded to GPU for (const auto& op : constants) { bool is_supported = true; + for (size_t i = 0; i < op->get_output_size(); i++) { auto outTensors = op->get_output_target_inputs(i); for (auto& t : outTensors) { auto output = t.get_node(); const auto& name = output->get_friendly_name(); - if (!InferenceEngine::details::contains(supported, name)) { + if (!InferenceEngine::details::contains(supported, name) && + !InferenceEngine::details::contains(supportedNotOriginal, name)) { is_supported = false; break; } } } if (is_supported) { - supported.emplace(op->get_friendly_name()); + if (InferenceEngine::details::contains(originalOpNames, op->get_friendly_name())) + supported.emplace(op->get_friendly_name()); + for (auto&& fusedLayerName : ngraph::getFusedNamesVector(op)) + if (InferenceEngine::details::contains(originalOpNames, fusedLayerName)) + supported.emplace(fusedLayerName); } } diff --git a/src/plugins/intel_myriad/common/src/ngraph/query_network.cpp b/src/plugins/intel_myriad/common/src/ngraph/query_network.cpp index c2290148392..459e0270a8a 100644 --- a/src/plugins/intel_myriad/common/src/ngraph/query_network.cpp +++ b/src/plugins/intel_myriad/common/src/ngraph/query_network.cpp @@ -47,6 +47,7 @@ InferenceEngine::QueryNetworkResult getQueryNetwork(const InferenceEngine::CNNNe } }; + for (InferenceEngine::details::CNNNetworkIterator itLayer{convertedNetwork}; itLayer != InferenceEngine::details::CNNNetworkIterator(); itLayer++) { @@ -55,9 +56,25 @@ InferenceEngine::QueryNetworkResult getQueryNetwork(const InferenceEngine::CNNNe continue; } + bool isSupported = false; + bool wasNodeAlreadyChecked = false; + if (InferenceEngine::details::contains(originalOps, fusedNode->get_friendly_name())) { + isSupported = isLayerSupported(itLayer); + wasNodeAlreadyChecked = true; + if (isSupported) { + supported.emplace(fusedNode->get_friendly_name()); + } else { + unsupported.emplace(fusedNode->get_friendly_name()); + } + } + for (auto& fusedLayerName : ngraph::getFusedNamesVector(fusedNode)) { if (InferenceEngine::details::contains(originalOps, fusedLayerName)) { - if (isLayerSupported(itLayer)) { + if (!wasNodeAlreadyChecked) { + isSupported = isLayerSupported(itLayer); + wasNodeAlreadyChecked = true; + } + if (isSupported) { supported.emplace(fusedLayerName); } else { unsupported.emplace(fusedLayerName); diff --git a/src/plugins/intel_myriad/myriad_plugin/myriad_plugin.cpp b/src/plugins/intel_myriad/myriad_plugin/myriad_plugin.cpp index 9a564c619d1..b71f4472a2a 100644 --- a/src/plugins/intel_myriad/myriad_plugin/myriad_plugin.cpp +++ b/src/plugins/intel_myriad/myriad_plugin/myriad_plugin.cpp @@ -180,9 +180,16 @@ QueryNetworkResult Engine::QueryNetwork( if (auto function = supportedNetwork.getFunction()) { auto clonedNetwork = cloneNetwork(supportedNetwork); + auto clonedFunction = clonedNetwork.getFunction(); auto convertedNetwork = vpu::FrontEnd::convertNetwork(clonedNetwork); QueryNetworkResult supportedRes = getQueryNetwork(clonedNetwork, function, GetName(), supportedLayers); + auto removedNodeNames = GetRemovedNodes(function, clonedFunction); + + for (const auto& layer : removedNodeNames) { + res.supportedLayersMap.emplace(layer, GetName()); + } + for (const auto& layer : supportedRes.supportedLayersMap) { res.supportedLayersMap.insert(layer); } diff --git a/src/tests/functional/plugin/cpu/behavior/hetero_cpu_query_network.cpp b/src/tests/functional/plugin/cpu/behavior/hetero_cpu_query_network.cpp new file mode 100644 index 00000000000..323cc2d2857 --- /dev/null +++ b/src/tests/functional/plugin/cpu/behavior/hetero_cpu_query_network.cpp @@ -0,0 +1,22 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/plugin/hetero_query_network.hpp" + +using namespace HeteroTests; + +namespace HeteroTests { + +TEST_P(HeteroQueryNetworkTest, HeteroSinglePlugin) { + std::string deviceName = GetParam(); + RunTest(deviceName); +} + +INSTANTIATE_TEST_CASE_P( + HeteroCpu, + HeteroQueryNetworkTest, + ::testing::Values( + std::string("HETERO:CPU"))); + +} // namespace HeteroTests diff --git a/src/tests/functional/plugin/gpu/behavior/hetero_gpu_query_network.cpp b/src/tests/functional/plugin/gpu/behavior/hetero_gpu_query_network.cpp new file mode 100644 index 00000000000..a04f8b80f66 --- /dev/null +++ b/src/tests/functional/plugin/gpu/behavior/hetero_gpu_query_network.cpp @@ -0,0 +1,22 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/plugin/hetero_query_network.hpp" + +using namespace HeteroTests; + +namespace HeteroTests { + +TEST_P(HeteroQueryNetworkTest, HeteroSinglePlugin) { + std::string deviceName = GetParam(); + RunTest(deviceName); +} + +INSTANTIATE_TEST_CASE_P( + HeteroGpu, + HeteroQueryNetworkTest, + ::testing::Values( + std::string("HETERO:GPU"))); + +} // namespace HeteroTests diff --git a/src/tests/functional/plugin/shared/include/behavior/plugin/hetero_query_network.hpp b/src/tests/functional/plugin/shared/include/behavior/plugin/hetero_query_network.hpp new file mode 100644 index 00000000000..1ea0cefa558 --- /dev/null +++ b/src/tests/functional/plugin/shared/include/behavior/plugin/hetero_query_network.hpp @@ -0,0 +1,466 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include "common_test_utils/test_common.hpp" + + +#include +#include + +using namespace InferenceEngine; + +namespace HeteroTests { + +class HeteroQueryNetworkTest : public ::testing::TestWithParam { +public: + void RunTest(std::string& deviceName) { + ASSERT_GT(deviceName.size(), 0); + + //this model is a subgraph of "ctpn" model from omz + std::string model = R"V0G0N( + + + + + + + 1 + 37 + 370 + 2 + + + + + + + + 4 + + + + + + + 1 + 37 + 370 + 2 + + + 4 + + + + + 1 + 2 + 37 + 370 + + + + + + + + 4 + + + + + + + 1 + 2 + 37 + 370 + + + 4 + + + + + 1 + 37 + 370 + 2 + + + + + + + + 1 + 2 + 37 + 370 + + + + + 4 + + + + + + + + 4 + + + + + + + + + + + + + + 4 + + + 4 + + + + + + 4 + + + + + + + + 1 + + + + + + + + 1 + + + + + + + + 1 + + + + + + + + 4 + + + 1 + + + 1 + + + 1 + + + + + + + + + + + 1 + + + + + + + + 1 + + + + + 1 + + + + + + + + 1 + + + + + + + + 1 + + + + + + + + 1 + + + + + + + + 4 + + + 1 + + + 1 + + + 1 + + + + + + + + + + + 1 + + + + + + + + 1 + + + + + 1 + + + + + + + + 1 + + + + + + + + 1 + + + + + + + + 1 + + + 1 + + + 1 + + + 1 + + + + + 4 + + + + + + + + 4 + + + + + 4 + + + + + + + + 1 + 37 + 370 + 2 + + + 4 + + + + + 1 + 37 + 37 + 20 + + + + + + + 1 + 37 + 37 + 20 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +)V0G0N"; + InferenceEngine::Core ie; + Blob::Ptr weights; + + weights = make_shared_blob(TensorDesc(Precision::U8, {160}, InferenceEngine::Layout::C)); + weights->allocate(); + + auto *dataI64 = weights->buffer().as(); + auto *dataI32 = weights->buffer().as(); + dataI64[0] = 0; + dataI64[1] = 3; + dataI64[2] = 1; + dataI64[3] = 2; + + dataI64[4] = 0; + dataI64[5] = 2; + dataI64[6] = 3; + dataI64[7] = 1; + + dataI32[16] = 0; + dataI32[17] = 2; + dataI32[18] = 3; + dataI32[19] = 1; + + dataI64[10] = 0; + + dataI64[11] = 0; + dataI64[12] = 1; + dataI64[13] = 1; + + dataI64[14] = 0; + + dataI64[15] = 1; + dataI64[16] = 2; + dataI64[17] = 1; + + dataI64[18] = 0; + + dataI32[38] = 0xFFFFFFFF; + dataI32[39] = 20; + + auto network = ie.ReadNetwork(model, weights); + + QueryNetworkResult result; + OV_ASSERT_NO_THROW(result = ie.QueryNetwork(network, deviceName)); + ASSERT_EQ(27, result.supportedLayersMap.size()); + + std::set checkNames = {"input", + "rpn_cls_prob/Transpose7580/value758213165", + "rpn_cls_prob/Transpose7580", + "rpn_cls_prob/Transpose/value756213066", + "rpn_cls_prob/Transpose", + "Shape_2", + "Shape_2/GatherNCHWtoNHWC_input_port_1/value778413036", + "Shape_2/GatherNCHWtoNHWC_input_port_2/value778613297", + "Shape_2/GatherNCHWtoNHWC", + "strided_slice_6/stack", + "strided_slice_6/stack_1", + "strided_slice_6/stack_2", + "strided_slice_6", + "Reshape_2/shape/Unsqueeze_input_port_1/value", + "Reshape_2/shape/Unsqueeze", + "strided_slice_7/stack", + "strided_slice_7/stack_1", + "strided_slice_7/stack_2", + "strided_slice_7", + "Reshape_2/shape/Unsqueeze531_input_port_1/value", + "Reshape_2/shape/Unsqueeze531", + "Reshape_2/shape/Unsqueeze533", + "Reshape_2/shape/Unsqueeze535", + "Reshape_2/shape", + "Reshape_2/Cast_1", + "Reshape_2", + "Reshape_2:0"}; + + for (auto&& name : checkNames) + EXPECT_NE(result.supportedLayersMap.find(name), result.supportedLayersMap.end()); + } +}; + +} // namespace HeteroTests From 78285f9db4f20ac5dfa4a0950613cf617221642f Mon Sep 17 00:00:00 2001 From: Anton Pankratov Date: Thu, 31 Mar 2022 07:36:13 +0300 Subject: [PATCH 14/39] Added ov::NotImplemented Exception (#11124) * Added ov::NotImplemented Exception * add ie namespace * Try fix --- src/core/include/openvino/core/except.hpp | 8 ++++++ ...executable_network_thread_safe_default.hpp | 2 +- src/inference/src/cpp/exception2status.hpp | 5 ++-- .../src/cpp/ie_infer_async_request_base.hpp | 10 ++++--- src/inference/src/cpp/ie_infer_request.cpp | 2 +- .../ie_iexecutable_network_internal.cpp | 2 +- .../interface/ie_iinfer_request_internal.cpp | 2 +- src/inference/src/ie_common.cpp | 26 +++++++++---------- src/plugins/auto/executable_network.cpp | 2 +- src/plugins/intel_cpu/src/node.cpp | 4 +-- 10 files changed, 37 insertions(+), 26 deletions(-) diff --git a/src/core/include/openvino/core/except.hpp b/src/core/include/openvino/core/except.hpp index cc8d3c8038d..1e8fdcf2dc5 100644 --- a/src/core/include/openvino/core/except.hpp +++ b/src/core/include/openvino/core/except.hpp @@ -45,6 +45,13 @@ private: const std::string& context_info, const std::string& explanation); }; + +/// Exception class to be thrown on not implemented code +class OPENVINO_API NotImplemented : public AssertFailure { +public: + NotImplemented(const CheckLocInfo& check_loc_info, const std::string& context_info, const std::string& explanation) + : AssertFailure(check_loc_info, context_info, explanation) {} +}; } // namespace ov // @@ -139,6 +146,7 @@ private: /// \throws ::ov::AssertFailure if the macro is executed. #define OPENVINO_UNREACHABLE(...) OPENVINO_ASSERT(false, "Unreachable: ", __VA_ARGS__) #define OPENVINO_ASSERT_HELPER(exc_class, ctx, ...) CALL_OVERLOAD(OPENVINO_ASSERT_HELPER, exc_class, ctx, __VA_ARGS__) +#define OPENVINO_NOT_IMPLEMENTED OPENVINO_ASSERT_HELPER(::ov::NotImplemented, "", false, "Not Implemented", "") #define GLUE(x, y) x y diff --git a/src/inference/dev_api/cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp b/src/inference/dev_api/cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp index 97f8ad370d7..56c83cbbd67 100644 --- a/src/inference/dev_api/cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp +++ b/src/inference/dev_api/cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp @@ -62,7 +62,7 @@ protected: InferenceEngine::IInferRequestInternal::Ptr syncRequestImpl; try { syncRequestImpl = this->CreateInferRequestImpl(_parameters, _results); - } catch (const NotImplemented&) { + } catch (const InferenceEngine::NotImplemented&) { } if (!syncRequestImpl) { syncRequestImpl = this->CreateInferRequestImpl(_networkInputs, _networkOutputs); diff --git a/src/inference/src/cpp/exception2status.hpp b/src/inference/src/cpp/exception2status.hpp index 85b90873b9b..8af89b10c0b 100644 --- a/src/inference/src/cpp/exception2status.hpp +++ b/src/inference/src/cpp/exception2status.hpp @@ -13,8 +13,9 @@ #include "description_buffer.hpp" namespace InferenceEngine { -#define CATCH_IE_EXCEPTION_TO_STATUS(StatusCode, ExceptionType) catch (const ExceptionType& ex) { \ - return InferenceEngine::DescriptionBuffer(StatusCode, resp) << ex.what(); \ +#define CATCH_IE_EXCEPTION_TO_STATUS(StatusCode, ExceptionType) \ +catch (const InferenceEngine::ExceptionType& ex) { \ + return InferenceEngine::DescriptionBuffer(StatusCode, resp) << ex.what(); \ } #define CATCH_IE_EXCEPTIONS_TO_STATUS \ diff --git a/src/inference/src/cpp/ie_infer_async_request_base.hpp b/src/inference/src/cpp/ie_infer_async_request_base.hpp index ec951977944..a5ed56aec57 100644 --- a/src/inference/src/cpp/ie_infer_async_request_base.hpp +++ b/src/inference/src/cpp/ie_infer_async_request_base.hpp @@ -16,8 +16,9 @@ namespace InferenceEngine { -#define CATCH_IE_EXCEPTION_TO_STATUS_NO_RESP(StatusCode, ExceptionType) catch (const ExceptionType& ex) { \ - return InferenceEngine::DescriptionBuffer(StatusCode) << ex.what(); \ +#define CATCH_IE_EXCEPTION_TO_STATUS_NO_RESP(StatusCode, ExceptionType) \ +catch (const InferenceEngine::ExceptionType& ex) { \ + return InferenceEngine::DescriptionBuffer(StatusCode) << ex.what(); \ } #define CATCH_IE_EXCEPTIONS_TO_STATUS_NO_RESP \ @@ -50,8 +51,9 @@ namespace InferenceEngine { return InferenceEngine::DescriptionBuffer(UNEXPECTED); \ } -#define CATCH_IE_EXCEPTION_CALL_RETURN_STATUS(StatusCode, ExceptionType) catch (const ExceptionType& ex) { \ - return InferenceEngine::DescriptionBuffer(StatusCode, resp) << ex.what(); \ +#define CATCH_IE_EXCEPTION_CALL_RETURN_STATUS(StatusCode, ExceptionType) \ +catch (const InferenceEngine::ExceptionType& ex) { \ + return InferenceEngine::DescriptionBuffer(StatusCode, resp) << ex.what(); \ } #define CATCH_IE_EXCEPTIONS_CALL_RETURN_STATUS \ diff --git a/src/inference/src/cpp/ie_infer_request.cpp b/src/inference/src/cpp/ie_infer_request.cpp index a565f78cf3c..2e24b76d9eb 100644 --- a/src/inference/src/cpp/ie_infer_request.cpp +++ b/src/inference/src/cpp/ie_infer_request.cpp @@ -134,7 +134,7 @@ void InferRequest::SetCompletionCallbackImpl(std::function callbackToSet } #define CATCH_IE_EXCEPTION_RETURN(StatusCode, ExceptionType) \ - catch (const ExceptionType&) { \ + catch (const ::InferenceEngine::ExceptionType&) { \ return StatusCode; \ } diff --git a/src/inference/src/cpp_interfaces/interface/ie_iexecutable_network_internal.cpp b/src/inference/src/cpp_interfaces/interface/ie_iexecutable_network_internal.cpp index 0b8610a9cf1..c8968cea28a 100644 --- a/src/inference/src/cpp_interfaces/interface/ie_iexecutable_network_internal.cpp +++ b/src/inference/src/cpp_interfaces/interface/ie_iexecutable_network_internal.cpp @@ -61,7 +61,7 @@ std::shared_ptr IExecutableNetworkInternal::CreateInferRe std::shared_ptr asyncRequestImpl; try { asyncRequestImpl = CreateInferRequestImpl(_parameters, _results); - } catch (const NotImplemented&) { + } catch (const InferenceEngine::NotImplemented&) { } if (!asyncRequestImpl) asyncRequestImpl = CreateInferRequestImpl(_networkInputs, _networkOutputs); diff --git a/src/inference/src/cpp_interfaces/interface/ie_iinfer_request_internal.cpp b/src/inference/src/cpp_interfaces/interface/ie_iinfer_request_internal.cpp index 4a0f40d87b1..d478b2302cb 100644 --- a/src/inference/src/cpp_interfaces/interface/ie_iinfer_request_internal.cpp +++ b/src/inference/src/cpp_interfaces/interface/ie_iinfer_request_internal.cpp @@ -312,7 +312,7 @@ void IInferRequestInternal::convertBatchedInputBlob(const std::string& name, con if (net) { remote_context = net->GetContext(); } - } catch (const NotImplemented&) { + } catch (const InferenceEngine::NotImplemented&) { } if (remote_context) { mem_blob = remote_context->CreateHostBlob(batched_desc); diff --git a/src/inference/src/ie_common.cpp b/src/inference/src/ie_common.cpp index 2073a314eb6..e7961bef0e8 100644 --- a/src/inference/src/ie_common.cpp +++ b/src/inference/src/ie_common.cpp @@ -45,31 +45,31 @@ namespace details { void Rethrow() { try { throw; - } catch (const GeneralError& e) { + } catch (const InferenceEngine::GeneralError& e) { throw e; - } catch (const NotImplemented& e) { + } catch (const InferenceEngine::NotImplemented& e) { throw e; - } catch (const NetworkNotLoaded& e) { + } catch (const InferenceEngine::NetworkNotLoaded& e) { throw e; - } catch (const ParameterMismatch& e) { + } catch (const InferenceEngine::ParameterMismatch& e) { throw e; - } catch (const NotFound& e) { + } catch (const InferenceEngine::NotFound& e) { throw e; - } catch (const OutOfBounds& e) { + } catch (const InferenceEngine::OutOfBounds& e) { throw e; - } catch (const Unexpected& e) { + } catch (const InferenceEngine::Unexpected& e) { throw e; - } catch (const RequestBusy& e) { + } catch (const InferenceEngine::RequestBusy& e) { throw e; - } catch (const ResultNotReady& e) { + } catch (const InferenceEngine::ResultNotReady& e) { throw e; - } catch (const NotAllocated& e) { + } catch (const InferenceEngine::NotAllocated& e) { throw e; - } catch (const InferNotStarted& e) { + } catch (const InferenceEngine::InferNotStarted& e) { throw e; - } catch (const NetworkNotRead& e) { + } catch (const InferenceEngine::NetworkNotRead& e) { throw e; - } catch (const InferCancelled& e) { + } catch (const InferenceEngine::InferCancelled& e) { throw e; } catch (const std::exception& e) { IE_THROW() << e.what(); diff --git a/src/plugins/auto/executable_network.cpp b/src/plugins/auto/executable_network.cpp index 36b2a828d4f..2c7c624fbdd 100644 --- a/src/plugins/auto/executable_network.cpp +++ b/src/plugins/auto/executable_network.cpp @@ -644,7 +644,7 @@ std::shared_ptr MultiDeviceExecutableNetwork::Ge const auto& n = _networksPerDevice.at(device.deviceName); try { return n->GetContext(); - } catch (const NotImplemented&) {} + } catch (const InferenceEngine::NotImplemented&) {} } IE_THROW(NotImplemented) << "None of the devices in the MULTI device has an associated remote context." << " Current list of devices allowed via the DEVICE_PRIORITIES config: " << devices_names; diff --git a/src/plugins/intel_cpu/src/node.cpp b/src/plugins/intel_cpu/src/node.cpp index 0d355a62a5d..a6b75d6d89f 100644 --- a/src/plugins/intel_cpu/src/node.cpp +++ b/src/plugins/intel_cpu/src/node.cpp @@ -1217,7 +1217,7 @@ Node* Node::NodesFactory::create(const std::shared_ptr& op, const if (ol != nullptr && ol->created(extMgr)) newNode = ol.release(); } catch (const InferenceEngine::Exception& ex) { - if (dynamic_cast(&ex) != nullptr) { + if (dynamic_cast(&ex) != nullptr) { errorMessage += getExceptionDescWithoutStatus(ex); } else { throw; @@ -1231,7 +1231,7 @@ Node* Node::NodesFactory::create(const std::shared_ptr& op, const if (ol != nullptr && ol->created(extMgr)) newNode = ol.release(); } catch (const InferenceEngine::Exception& ex) { - if (dynamic_cast(&ex) != nullptr) { + if (dynamic_cast(&ex) != nullptr) { const auto currErrorMess = getExceptionDescWithoutStatus(ex); if (!currErrorMess.empty()) errorMessage += errorMessage.empty() ? currErrorMess : "\n" + currErrorMess; From 3e58ccbce7758eb797c0b972813ef111ff63a2ce Mon Sep 17 00:00:00 2001 From: Ilya Churaev Date: Thu, 31 Mar 2022 07:47:49 +0300 Subject: [PATCH 15/39] Fixed evaluate for ov::Tensor (#11354) * Fixed evaluate for ov::Tensor * Fixed old ops with EvaluationContext --- docs/template_extension/new/identity.cpp | 2 +- src/core/src/node.cpp | 4 +- src/core/tests/extension.cpp | 59 ++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/docs/template_extension/new/identity.cpp b/docs/template_extension/new/identity.cpp index 442753c1f5c..9443d394289 100644 --- a/docs/template_extension/new/identity.cpp +++ b/docs/template_extension/new/identity.cpp @@ -38,7 +38,7 @@ bool Identity::evaluate(ov::TensorVector& outputs, const ov::TensorVector& input auto in = inputs[0]; auto out = outputs[0]; out.set_shape(in.get_shape()); - memcpy(out.data(), in.data(), in.get_size()); + memcpy(out.data(), in.data(), in.get_byte_size()); return true; } diff --git a/src/core/src/node.cpp b/src/core/src/node.cpp index d8164ba336e..68bbbd8b841 100644 --- a/src/core/src/node.cpp +++ b/src/core/src/node.cpp @@ -743,13 +743,15 @@ bool ov::Node::evaluate(ov::TensorVector& output_values, const ov::TensorVector& bool ov::Node::evaluate(ov::TensorVector& output_values, const ov::TensorVector& input_values, const ov::EvaluationContext& evaluationContext) const { + // Call evaluate for old implementation with EvaluationContext HostTensorVector output = create_tmp_tensors(output_values); HostTensorVector input = create_tmp_tensors(input_values); OPENVINO_SUPPRESS_DEPRECATED_START bool sts = evaluate(output, input, evaluationContext); OPENVINO_SUPPRESS_DEPRECATED_END update_output_tensors(output_values, output); - return sts; + // Call evaluate for ov::Tensor if op doesn't have evaluate with EvaluationContext + return sts ? sts : evaluate(output_values, input_values); } bool ov::Node::evaluate_lower(ov::TensorVector& output_values) const { diff --git a/src/core/tests/extension.cpp b/src/core/tests/extension.cpp index 187fa3e9481..4bc5961fd68 100644 --- a/src/core/tests/extension.cpp +++ b/src/core/tests/extension.cpp @@ -8,6 +8,7 @@ #include "openvino/core/graph_util.hpp" #include "openvino/core/op_extension.hpp" +#include "openvino/opsets/opset9.hpp" #include "openvino/util/file_util.hpp" #include "so_extension.hpp" @@ -36,3 +37,61 @@ TEST(extension, load_extension_and_cast) { EXPECT_NE(nullptr, std::dynamic_pointer_cast(extensions[0])); extensions.clear(); } + +namespace { +class DummyAdapter : public ov::AttributeVisitor { +public: + void on_adapter(const std::string& name, ov::ValueAccessor& adapter) override {} +}; +} // namespace + +TEST(extension, create_model_from_extension) { + std::vector so_extensions = ov::detail::load_extensions(get_extension_path()); + ASSERT_LE(1, so_extensions.size()); + std::vector extensions; + std::vector> so; + for (const auto& ext : so_extensions) { + if (auto so_ext = std::dynamic_pointer_cast(ext)) { + extensions.emplace_back(so_ext->extension()); + so.emplace_back(so_ext->shared_object()); + } + } + so_extensions.clear(); + EXPECT_LE(1, extensions.size()); + auto op_extension = std::dynamic_pointer_cast(extensions[0]); + EXPECT_NE(nullptr, op_extension); + { + // Create model to check evaluate for custom operation + std::shared_ptr model; + { + auto parameter = std::make_shared(ov::element::i32, ov::Shape{1, 2, 2, 2}); + + DummyAdapter visitor; + + auto outputs = op_extension->create(ov::OutputVector{parameter}, visitor); + + EXPECT_EQ(1, outputs.size()); + EXPECT_NE(nullptr, outputs[0].get_node()); + const std::string ref_name = "Identity"; + EXPECT_EQ(ref_name, outputs[0].get_node()->get_type_info().name); + auto result = std::make_shared(outputs[0]); + model = std::make_shared(ov::ResultVector{result}, ov::ParameterVector{parameter}); + } + + auto fill_tensor = [](ov::Tensor& tensor) { + int32_t* data = tensor.data(); + for (size_t i = 0; i < tensor.get_size(); i++) + data[i] = i; + }; + + ov::TensorVector inputs; + inputs.emplace_back(ov::Tensor(ov::element::i32, ov::Shape{1, 2, 2, 2})); + fill_tensor(*inputs.begin()); + ov::TensorVector outputs; + outputs.emplace_back(ov::Tensor(ov::element::i32, ov::Shape{1, 2, 2, 2})); + EXPECT_NE(std::memcmp(inputs.begin()->data(), outputs.begin()->data(), inputs.begin()->get_byte_size()), 0); + model->evaluate(outputs, inputs); + EXPECT_EQ(std::memcmp(inputs.begin()->data(), outputs.begin()->data(), inputs.begin()->get_byte_size()), 0); + } + extensions.clear(); +} From f5f93cfbeb8763b308d38f26be18d72d87e61986 Mon Sep 17 00:00:00 2001 From: Vladimir Paramuzov Date: Thu, 31 Mar 2022 08:54:05 +0300 Subject: [PATCH 16/39] [GPU] Strided slice primitive params update (#11339) --- .../intel_gpu/primitives/strided_slice.hpp | 16 +++++------ .../src/graph/impls/ocl/strided_slice.cpp | 24 ++++++++++++++--- .../intel_gpu/src/graph/strided_slice.cpp | 3 ++- .../src/plugin/ops/strided_slice.cpp | 27 ++++--------------- .../test_cases/strided_slice_gpu_test.cpp | 16 +++++------ 5 files changed, 43 insertions(+), 43 deletions(-) diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/strided_slice.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/strided_slice.hpp index a1c02bfbed3..9f39ee52d85 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/primitives/strided_slice.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/primitives/strided_slice.hpp @@ -38,10 +38,10 @@ struct strided_slice : public primitive_base { const primitive_id& begin_id, const primitive_id& end_id, const primitive_id& strides_id, - std::vector begin_mask, - std::vector end_mask, - std::vector new_axis_mask, - std::vector shrink_axis_mask, + std::vector begin_mask, + std::vector end_mask, + std::vector new_axis_mask, + std::vector shrink_axis_mask, const ov::Shape out_size, const primitive_id& ext_prim_id = "", const padding& output_padding = padding()) @@ -53,13 +53,13 @@ struct strided_slice : public primitive_base { out_size(out_size) {} /// @brief Array of bits, that provide replace begin[i] to max possible range in that dimension. - std::vector begin_mask; + std::vector begin_mask; /// @brief Array of bits, that provide replace end[i] to max possible range in that dimension. - std::vector end_mask; + std::vector end_mask; /// @brief Array of bits, that provide adding a new length 1 dimension at ith position in the output tensor. - std::vector new_axis_mask; + std::vector new_axis_mask; /// @brief Array of bits, that provide shrinks the dimensionality by 1, taking on the value at index begin[i]. - std::vector shrink_axis_mask; + std::vector shrink_axis_mask; /// @brief Size of output tensor ov::Shape out_size; }; diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/strided_slice.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/strided_slice.cpp index e558f0f9a0e..9e8aea479eb 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/strided_slice.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/strided_slice.cpp @@ -52,12 +52,28 @@ public: params.striding_params.push_back(sizes); } - params.end_mask = arg.get_primitive()->end_mask; + auto begin_mask_ = arg.get_primitive()->begin_mask; + auto end_mask_ = arg.get_primitive()->end_mask; + auto new_axis_mask_ = arg.get_primitive()->new_axis_mask; + auto shrink_axis_mask_ = arg.get_primitive()->shrink_axis_mask; + + std::vector begin_mask(begin_mask_.begin(), begin_mask_.end()); + std::vector end_mask(end_mask_.begin(), end_mask_.end()); + std::vector new_axis_mask(new_axis_mask_.begin(), new_axis_mask_.end()); + std::vector shrink_axis_mask(shrink_axis_mask_.begin(), shrink_axis_mask_.end()); + // Plugin requires inverted mask values. Consider changing primitive impl to be aligned with the spec. + for (auto& b : begin_mask) { + b = 1 - b; + } + for (auto& e : end_mask) { + e = 1 - e; + } + params.end_mask = end_mask; pad_vector_to_size(params.end_mask, dims_num, 1); - params.begin_mask = arg.get_primitive()->begin_mask; + params.begin_mask = begin_mask; pad_vector_to_size(params.begin_mask, dims_num, 1); - params.new_axis_mask = arg.get_primitive()->new_axis_mask; - params.shrink_axis_mask = arg.get_primitive()->shrink_axis_mask; + params.new_axis_mask = new_axis_mask; + params.shrink_axis_mask = shrink_axis_mask; pad_vector_to_size(params.shrink_axis_mask, dims_num, 0); std::vector logical_dims = params.inputs[0].LogicalDims(); diff --git a/src/plugins/intel_gpu/src/graph/strided_slice.cpp b/src/plugins/intel_gpu/src/graph/strided_slice.cpp index 665463886d0..aeee48bcb68 100644 --- a/src/plugins/intel_gpu/src/graph/strided_slice.cpp +++ b/src/plugins/intel_gpu/src/graph/strided_slice.cpp @@ -20,7 +20,8 @@ layout strided_slice_inst::calc_output_layout(strided_slice_node const& node) { auto desc = node.get_primitive(); auto input_layout = node.input(0).get_output_layout(); auto output_format = format::get_default_format(desc->out_size.size()); - std::vector dims_converted(desc->out_size.begin(), desc->out_size.end()); + auto out_shape = desc->out_size; + std::vector dims_converted(out_shape.begin(), out_shape.end()); // extend shape to 4d for (size_t i = dims_converted.size(); i < 4; i++) { dims_converted.push_back(1); diff --git a/src/plugins/intel_gpu/src/plugin/ops/strided_slice.cpp b/src/plugins/intel_gpu/src/plugin/ops/strided_slice.cpp index 5941140c961..595d84c97f6 100644 --- a/src/plugins/intel_gpu/src/plugin/ops/strided_slice.cpp +++ b/src/plugins/intel_gpu/src/plugin/ops/strided_slice.cpp @@ -234,33 +234,16 @@ static void CreateStridedSliceOp(Program& p, const std::shared_ptrget_end_mask(); - auto begin_mask_ = op->get_begin_mask(); - auto new_axis_mask_ = op->get_new_axis_mask(); - auto shrink_axis_mask_ = op->get_shrink_axis_mask(); - std::vector begin_mask(begin_mask_.begin(), begin_mask_.end()); - std::vector end_mask(end_mask_.begin(), end_mask_.end()); - std::vector new_axis_mask(new_axis_mask_.begin(), new_axis_mask_.end()); - std::vector shrink_axis_mask(shrink_axis_mask_.begin(), shrink_axis_mask_.end()); - - // Plugin requires inverted mask values. Consider changing primitive impl to be aligned with the spec. - for (auto& b : begin_mask) { - b = 1 - b; - } - for (auto& e : end_mask) { - e = 1 - e; - } - auto stridedSlicePrim = cldnn::strided_slice(layerName, inputPrimitives[0], inputPrimitives[1], inputPrimitives[2], inputPrimitives[3], - begin_mask, - end_mask, - new_axis_mask, - shrink_axis_mask, - op->get_output_shape(0), + op->get_begin_mask(), + op->get_end_mask(), + op->get_new_axis_mask(), + op->get_shrink_axis_mask(), + op->get_output_partial_shape(0).to_shape(), op->get_friendly_name()); p.AddPrimitive(stridedSlicePrim); diff --git a/src/plugins/intel_gpu/tests/test_cases/strided_slice_gpu_test.cpp b/src/plugins/intel_gpu/tests/test_cases/strided_slice_gpu_test.cpp index 9932b31f5ef..5c3b1fec54c 100644 --- a/src/plugins/intel_gpu/tests/test_cases/strided_slice_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/test_cases/strided_slice_gpu_test.cpp @@ -159,7 +159,7 @@ TEST(strided_slice_gpu_f32_i32, test_2x2x2x2_ignore) { topology.add(data("input2", begin)); topology.add(data("input3", end)); topology.add(data("input4", strides)); - topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {0, 0, 0, 0}, {0, 0, 0, 0}, {}, {}, {2, 2, 2, 2})); + topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {1, 1, 1, 1}, {1, 1, 1, 1}, {}, {}, {2, 2, 2, 2})); network network(engine, topology); @@ -218,7 +218,7 @@ TEST(strided_slice_gpu_f32_i64, test_2x2x2x2_ignore) { topology.add(data("input2", begin)); topology.add(data("input3", end)); topology.add(data("input4", strides)); - topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {0, 0, 0, 0}, {0, 0, 0, 0}, {}, {}, {2, 2, 2, 2})); + topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {1, 1, 1, 1}, {1, 1, 1, 1}, {}, {}, {2, 2, 2, 2})); network network(engine, topology); @@ -393,7 +393,7 @@ TEST(strided_slice_gpu_f32_i32, test_2x2x4x3_stride) { topology.add(data("input2", begin)); topology.add(data("input3", end)); topology.add(data("input4", strides)); - topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {0, 0, 0, 0}, {0, 0, 0, 0}, {}, {}, {2, 2, 2, 3})); + topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {1, 1, 1, 1}, {1, 1, 1, 1}, {}, {}, {2, 2, 2, 3})); network network(engine, topology); @@ -456,7 +456,7 @@ TEST(strided_slice_gpu_f32_i64, test_2x2x4x3_stride) { topology.add(data("input2", begin)); topology.add(data("input3", end)); topology.add(data("input4", strides)); - topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {0, 0, 0, 0}, {0, 0, 0, 0}, {}, {}, {2, 2, 2, 3})); + topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {1, 1, 1, 1}, {1, 1, 1, 1}, {}, {}, {2, 2, 2, 3})); network network(engine, topology); @@ -532,7 +532,7 @@ TEST(strided_slice_gpu_f32_i32, test_2x2x4x4_part_stride) { topology.add(data("input2", begin)); topology.add(data("input3", end)); topology.add(data("input4", strides)); - topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {1, 0, 0, 1}, {}, {}, {}, {1, 2, 4, 2})); + topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {0, 1, 1, 0}, {}, {}, {}, {1, 2, 4, 2})); network network(engine, topology); @@ -615,7 +615,7 @@ TEST(strided_slice_gpu_f32_i64, test_2x2x4x4_part_stride) { topology.add(data("input2", begin)); topology.add(data("input3", end)); topology.add(data("input4", strides)); - topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {1, 0, 0, 1}, {}, {}, {}, {1, 2, 4, 2})); + topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {0, 1, 1, 0}, {}, {}, {}, {1, 2, 4, 2})); network network(engine, topology); @@ -897,7 +897,7 @@ TEST(strided_slice_gpu_f32_i32, test_2x2x1x1) { topology.add(data("input2", begin)); topology.add(data("input3", end)); topology.add(data("input4", strides)); - topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {0,1}, {}, {}, {}, {2, 2, 1, 1})); + topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {1, 0}, {}, {}, {}, {2, 2, 1, 1})); network network(engine, topology); @@ -950,7 +950,7 @@ TEST(strided_slice_gpu_f32_i64, test_2x2x1x1) { topology.add(data("input2", begin)); topology.add(data("input3", end)); topology.add(data("input4", strides)); - topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {0,1}, {}, {}, {}, {2, 2, 1, 1})); + topology.add(strided_slice("strided_slice", "input", "input2", "input3", "input4", {1, 0}, {}, {}, {}, {2, 2, 1, 1})); network network(engine, topology); From 24a74672f6e2e81d0c44691a712cb388633cd90c Mon Sep 17 00:00:00 2001 From: Sergey Shlyapnikov Date: Thu, 31 Mar 2022 09:04:46 +0300 Subject: [PATCH 17/39] [GPU] Fix remote blobs tests (#11349) --- .../cldnn_remote_blob_tests.cpp | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/tests/functional/plugin/gpu/remote_blob_tests/cldnn_remote_blob_tests.cpp b/src/tests/functional/plugin/gpu/remote_blob_tests/cldnn_remote_blob_tests.cpp index 80ae17fbe6d..3ff0f73c03a 100644 --- a/src/tests/functional/plugin/gpu/remote_blob_tests/cldnn_remote_blob_tests.cpp +++ b/src/tests/functional/plugin/gpu/remote_blob_tests/cldnn_remote_blob_tests.cpp @@ -77,8 +77,10 @@ TEST_P(RemoteBlob_Test, smoke_canInputUserBlob) { auto ocl_instance = std::make_shared(ctx); cl_int err; - auto dims = net.getInputsInfo().begin()->second->getTensorDesc().getDims(); - size_t imSize = dims[1] * dims[2] * dims[3]; + auto desc = net.getInputsInfo().begin()->second->getTensorDesc(); + size_t imSize = std::accumulate(desc.getDims().begin(), desc.getDims().end(), + desc.getPrecision().size(), + std::multiplies()); cl::Buffer shared_buffer(ocl_instance->_context, CL_MEM_READ_WRITE, imSize, NULL, &err); { @@ -190,8 +192,10 @@ TEST_P(RemoteBlob_Test, smoke_canInputPluginRemoteBlob) { cl_context ctx = std::dynamic_pointer_cast(cldnn_context)->get(); auto ocl_instance = std::make_shared(ctx); - auto dims = net.getInputsInfo().begin()->second->getTensorDesc().getDims(); - size_t imSize = dims[1] * dims[2] * dims[3]; + auto desc = net.getInputsInfo().begin()->second->getTensorDesc(); + size_t imSize = std::accumulate(desc.getDims().begin(), desc.getDims().end(), + desc.getPrecision().size(), + std::multiplies()); Blob::Ptr shared_blob = make_shared_blob(net.getInputsInfo().begin()->second->getTensorDesc(), cldnn_context); shared_blob->allocate(); @@ -287,8 +291,12 @@ TEST_P(RemoteBlob_Test, smoke_canInferOnUserQueue_out_of_order) { auto out_desc = net.getOutputsInfo().begin()->second->getTensorDesc(); auto in_dims = in_desc.getDims(); auto out_dims = out_desc.getDims(); - size_t in_size = in_dims[1] * in_dims[2] * in_dims[3]; - size_t out_size = out_dims[1] * out_dims[2] * out_dims[3] * sizeof(float); + size_t in_size = std::accumulate(in_desc.getDims().begin(), in_desc.getDims().end(), + in_desc.getPrecision().size(), + std::multiplies()); + size_t out_size = std::accumulate(out_desc.getDims().begin(), out_desc.getDims().end(), + out_desc.getPrecision().size(), + std::multiplies()); // In this scenario we create shared OCL queue and run simple pre-process action and post-process action (buffer copies in both cases) // without calling thread blocks @@ -379,8 +387,12 @@ TEST_P(RemoteBlob_Test, smoke_canInferOnUserQueue_in_order) { auto out_desc = net.getOutputsInfo().begin()->second->getTensorDesc(); auto in_dims = in_desc.getDims(); auto out_dims = out_desc.getDims(); - size_t in_size = in_dims[1] * in_dims[2] * in_dims[3]; - size_t out_size = out_dims[1] * out_dims[2] * out_dims[3] * sizeof(float); + size_t in_size = std::accumulate(in_desc.getDims().begin(), in_desc.getDims().end(), + in_desc.getPrecision().size(), + std::multiplies()); + size_t out_size = std::accumulate(out_desc.getDims().begin(), out_desc.getDims().end(), + out_desc.getPrecision().size(), + std::multiplies()); // In this scenario we create shared OCL queue and run simple pre-process action and post-process action (buffer copies in both cases) // without calling thread blocks From 5c917cfaaacce33c2ed53cf1afb6c3f1dad818b3 Mon Sep 17 00:00:00 2001 From: Daria Mityagina Date: Thu, 31 Mar 2022 09:50:26 +0300 Subject: [PATCH 18/39] [ICV][XLink] - port XLink changes from mdk (#10212) -76384 Port changes from MDK --- .../intel_myriad/myriad_dependencies.cmake | 6 +- .../third_party/XLink/shared/include/XLink.h | 1 - .../XLink/shared/include/XLinkDispatcher.h | 14 +- .../shared/include/XLinkPrivateDefines.h | 6 + .../third_party/XLink/shared/src/XLinkData.c | 32 ++-- .../XLink/shared/src/XLinkDispatcher.c | 146 ++++++++++++------ .../XLink/shared/src/XLinkDispatcherImpl.c | 54 +++++-- 7 files changed, 185 insertions(+), 74 deletions(-) diff --git a/src/plugins/intel_myriad/myriad_dependencies.cmake b/src/plugins/intel_myriad/myriad_dependencies.cmake index 5bba4235ffb..bede57fa3fe 100644 --- a/src/plugins/intel_myriad/myriad_dependencies.cmake +++ b/src/plugins/intel_myriad/myriad_dependencies.cmake @@ -6,14 +6,14 @@ include_guard(GLOBAL) set(VPU_SUPPORTED_FIRMWARES usb-ma2x8x pcie-ma2x8x) set(VPU_SUPPORTED_FIRMWARES_HASH - "e65fcc1c6b0f3e9d814e53022c212ec0a2b83197a9df38badb298fb85ccf3acf" - "b11368fec2036d96fb703d2a40b171184fefe89f27e74a988ef1ca34260a2bc5") + "877c4e1616d14a94dd2764f4f32f1c1aa2180dcd64ad1823b31efdc3f56ad593" + "aabff3d817431792ef9e17056448979c2cdbb484ad4b0af9e68cb874ee10eef5") # # Default packages # -set(FIRMWARE_PACKAGE_VERSION 1875) +set(FIRMWARE_PACKAGE_VERSION 20220307_34) set(VPU_CLC_MA2X8X_VERSION "movi-cltools-20.09.2") # diff --git a/src/plugins/intel_myriad/third_party/XLink/shared/include/XLink.h b/src/plugins/intel_myriad/third_party/XLink/shared/include/XLink.h index 89dfcaf88db..76fb8bf8a69 100644 --- a/src/plugins/intel_myriad/third_party/XLink/shared/include/XLink.h +++ b/src/plugins/intel_myriad/third_party/XLink/shared/include/XLink.h @@ -205,7 +205,6 @@ XLinkError_t XLinkReadDataWithTimeout(streamId_t streamId, streamPacketDesc_t** /** * @brief Sends a package to initiate the writing of data to a remote stream with timeout in ms - * XLinkWriteDataWithTimeout is not fully supported yet. The XLinkWriteData method is called instead. * @warning Actual size of the written data is ALIGN_UP(size, 64) * @param[in] streamId – stream link Id obtained from XLinkOpenStream call * @param[in] buffer – data buffer to be transmitted diff --git a/src/plugins/intel_myriad/third_party/XLink/shared/include/XLinkDispatcher.h b/src/plugins/intel_myriad/third_party/XLink/shared/include/XLinkDispatcher.h index e51da860548..80abd2ba500 100644 --- a/src/plugins/intel_myriad/third_party/XLink/shared/include/XLinkDispatcher.h +++ b/src/plugins/intel_myriad/third_party/XLink/shared/include/XLinkDispatcher.h @@ -36,13 +36,13 @@ int DispatcherWaitEventComplete(xLinkDeviceHandle_t *deviceHandle, unsigned int char* TypeToStr(int type); int DispatcherUnblockEvent(eventId_t id, - xLinkEventType_t type, - streamId_t stream, - void *xlinkFD); -int DispatcherServeEvent(eventId_t id, - xLinkEventType_t type, - streamId_t stream, - void *xlinkFD); + xLinkEventType_t type, + streamId_t stream, + void *xlinkFD); +int DispatcherServeOrDropEvent(eventId_t id, + xLinkEventType_t type, + streamId_t stream, + void *xlinkFD); #ifdef __cplusplus } #endif diff --git a/src/plugins/intel_myriad/third_party/XLink/shared/include/XLinkPrivateDefines.h b/src/plugins/intel_myriad/third_party/XLink/shared/include/XLinkPrivateDefines.h index f2ed4f5660e..8588b050d03 100644 --- a/src/plugins/intel_myriad/third_party/XLink/shared/include/XLinkPrivateDefines.h +++ b/src/plugins/intel_myriad/third_party/XLink/shared/include/XLinkPrivateDefines.h @@ -84,6 +84,7 @@ typedef enum XLINK_CLOSE_STREAM_REQ, XLINK_PING_REQ, XLINK_RESET_REQ, + XLINK_DROP_REQ, XLINK_REQUEST_LAST, //note that is important to separate request and response XLINK_WRITE_RESP, @@ -94,6 +95,7 @@ typedef enum XLINK_CLOSE_STREAM_RESP, XLINK_PING_RESP, XLINK_RESET_RESP, + XLINK_DROP_RESP, XLINK_RESP_LAST, /*X_LINK_IPC related events*/ @@ -130,6 +132,8 @@ typedef struct xLinkEventHeader_t{ char streamName[MAX_STREAM_NAME_LENGTH]; streamId_t streamId; uint32_t size; + uint32_t dropped; + uint32_t canBeServed; union{ uint32_t raw; struct{ @@ -155,6 +159,8 @@ typedef struct xLinkEvent_t { (event).header.streamId = (in_streamId); \ (event).header.type = (in_type); \ (event).header.size = (in_size); \ + (event).header.dropped = 0; \ + (event).header.canBeServed = 1; \ (event).data = (in_data); \ (event).deviceHandle = (in_deviceHandle); \ } while(0) diff --git a/src/plugins/intel_myriad/third_party/XLink/shared/src/XLinkData.c b/src/plugins/intel_myriad/third_party/XLink/shared/src/XLinkData.c index df859e0c633..c9ef984aac0 100644 --- a/src/plugins/intel_myriad/third_party/XLink/shared/src/XLinkData.c +++ b/src/plugins/intel_myriad/third_party/XLink/shared/src/XLinkData.c @@ -182,8 +182,7 @@ XLinkError_t XLinkWriteDataWithTimeout(streamId_t streamId, const uint8_t* buffe XLINK_INIT_EVENT(event, streamId, XLINK_WRITE_REQ, size,(void*)buffer, link->deviceHandle); - mvLog(MVLOG_WARN,"XLinkWriteDataWithTimeout is not fully supported yet. The XLinkWriteData method is called instead. Desired timeout = %d\n", timeoutMs); - XLINK_RET_IF_FAIL(addEventWithPerf(&event, &opTime, XLINK_NO_RW_TIMEOUT)); + XLINK_RET_IF_FAIL(addEventWithPerf(&event, &opTime, timeoutMs)); if( glHandler->profEnable) { glHandler->profilingData.totalWriteBytes += size; @@ -322,10 +321,10 @@ XLinkError_t addEvent(xLinkEvent_t *event, unsigned int timeoutMs) TypeToStr(event->header.type), event->header.id, event->header.streamName); return X_LINK_ERROR; } + event->header.canBeServed = 1; if (timeoutMs != XLINK_NO_RW_TIMEOUT) { - ASSERT_XLINK(event->header.type == XLINK_READ_REQ); - xLinkDesc_t* link; + xLinkDesc_t* link = NULL; XLINK_RET_IF(getLinkByStreamId(event->header.streamId, &link)); if (DispatcherWaitEventComplete(&event->deviceHandle, timeoutMs)) // timeout reached @@ -333,17 +332,28 @@ XLinkError_t addEvent(xLinkEvent_t *event, unsigned int timeoutMs) streamDesc_t* stream = getStreamById(event->deviceHandle.xLinkFD, event->header.streamId); ASSERT_XLINK(stream); + event->header.dropped = 1; if (event->header.type == XLINK_READ_REQ) { - // XLINK_READ_REQ is a local event. It is safe to serve it. - // Limitations. - // Possible vulnerability in this mechanism: - // If we reach timeout with DispatcherWaitEventComplete and before - // we call DispatcherServeEvent, the event actually comes, - // and gets served by XLink stack and event semaphore is posted. - DispatcherServeEvent(event->header.id, XLINK_READ_REQ, stream->id, event->deviceHandle.xLinkFD); + event->header.canBeServed = 0; + XLINK_RET_IF(DispatcherServeOrDropEvent(event->header.id, XLINK_READ_REQ, stream->id, event->deviceHandle.xLinkFD)); + } + else if (event->header.type == XLINK_WRITE_REQ) + { + event->header.canBeServed = 0; + XLINK_RET_IF(DispatcherServeOrDropEvent(event->header.id, XLINK_WRITE_REQ, stream->id, event->deviceHandle.xLinkFD)); } releaseStream(stream); + if (event->header.type == XLINK_WRITE_REQ && event->header.dropped) + { + mvLog(MVLOG_ERROR,"event is dropped\n"); + xLinkEvent_t dropEvent = {0}; + dropEvent.header.streamId = EXTRACT_STREAM_ID(event->header.streamId); + XLINK_INIT_EVENT(dropEvent, event->header.streamId, XLINK_DROP_REQ, + 0, NULL, link->deviceHandle); + DispatcherAddEvent(EVENT_LOCAL, &dropEvent); + XLINK_RET_ERR_IF(DispatcherWaitEventComplete(&link->deviceHandle, XLINK_NO_RW_TIMEOUT), dropEvent.header.streamId); + } return X_LINK_TIMEOUT; } diff --git a/src/plugins/intel_myriad/third_party/XLink/shared/src/XLinkDispatcher.c b/src/plugins/intel_myriad/third_party/XLink/shared/src/XLinkDispatcher.c index fba304f548e..f38ed5dfe8d 100644 --- a/src/plugins/intel_myriad/third_party/XLink/shared/src/XLinkDispatcher.c +++ b/src/plugins/intel_myriad/third_party/XLink/shared/src/XLinkDispatcher.c @@ -50,6 +50,7 @@ typedef enum { EVENT_BLOCKED, EVENT_READY, EVENT_SERVED, + EVENT_DROPPED, } xLinkEventState_t; typedef struct xLinkEventPriv_t { @@ -446,6 +447,7 @@ char* TypeToStr(int type) case XLINK_CLOSE_STREAM_REQ: return "XLINK_CLOSE_STREAM_REQ"; case XLINK_PING_REQ: return "XLINK_PING_REQ"; case XLINK_RESET_REQ: return "XLINK_RESET_REQ"; + case XLINK_DROP_REQ: return "XLINK_DROP_REQ"; case XLINK_REQUEST_LAST: return "XLINK_REQUEST_LAST"; case XLINK_WRITE_RESP: return "XLINK_WRITE_RESP"; case XLINK_READ_RESP: return "XLINK_READ_RESP"; @@ -455,6 +457,7 @@ char* TypeToStr(int type) case XLINK_CLOSE_STREAM_RESP: return "XLINK_CLOSE_STREAM_RESP"; case XLINK_PING_RESP: return "XLINK_PING_RESP"; case XLINK_RESET_RESP: return "XLINK_RESET_RESP"; + case XLINK_DROP_RESP: return "XLINK_DROP_RESP"; case XLINK_RESP_LAST: return "XLINK_RESP_LAST"; default: break; @@ -462,6 +465,59 @@ char* TypeToStr(int type) return ""; } +int DispatcherServeOrDropEvent(eventId_t id, xLinkEventType_t type, streamId_t stream, void *xlinkFD) +{ + xLinkSchedulerState_t* curr = findCorrespondingScheduler(xlinkFD); + ASSERT_XLINK(curr != NULL); + + xLinkEventPriv_t* event; + int blocked = 0; + XLINK_RET_ERR_IF(pthread_mutex_lock(&(curr->queueMutex)) != 0, 1); + for (event = curr->lQueue.q; + event < curr->lQueue.q + MAX_EVENTS; + event++) + { + if (((event->packet.header.id == id || id == -1) + && event->packet.header.type == type + && event->packet.header.streamId == stream)) + { + blocked = event->isServed == EVENT_BLOCKED; + if (event->packet.header.type == XLINK_READ_REQ) { + mvLog(MVLOG_DEBUG,"served read request**************** %d %s\n", + (int)event->packet.header.id, + TypeToStr((int)event->packet.header.type)); + event->isServed = EVENT_SERVED; + event->packet.header.dropped = 0; + event->packet.header.canBeServed = 1; + XLINK_RET_ERR_IF(pthread_mutex_unlock(&(curr->queueMutex)) != 0, 1); + return 0; + } else if (event->packet.header.type == XLINK_WRITE_REQ) { + if (blocked) { + mvLog(MVLOG_DEBUG,"served write request**************** %d %s\n", + (int)event->packet.header.id, + TypeToStr((int)event->packet.header.type)); + event->isServed = EVENT_SERVED; + event->packet.header.dropped = 0; + event->packet.header.canBeServed = 1; + XLINK_RET_ERR_IF(pthread_mutex_unlock(&(curr->queueMutex)) != 0, 1); + return 0; + } else { + mvLog(MVLOG_DEBUG,"droped write request**************** %d %s\n", + (int)event->packet.header.id, + TypeToStr((int)event->packet.header.type)); + event->isServed = EVENT_DROPPED; + event->packet.header.dropped = 1; + event->packet.header.canBeServed = 1; + XLINK_RET_ERR_IF(pthread_mutex_unlock(&(curr->queueMutex)) != 0, 1); + return 0; + } + } + } + } + XLINK_RET_ERR_IF(pthread_mutex_unlock(&(curr->queueMutex)) != 0, 1); + return 0; +} + int DispatcherUnblockEvent(eventId_t id, xLinkEventType_t type, streamId_t stream, void *xlinkFD) { xLinkSchedulerState_t* curr = findCorrespondingScheduler(xlinkFD); @@ -499,33 +555,6 @@ int DispatcherUnblockEvent(eventId_t id, xLinkEventType_t type, streamId_t strea return 0; } -int DispatcherServeEvent(eventId_t id, xLinkEventType_t type, streamId_t stream, void *xlinkFD) -{ - xLinkSchedulerState_t* curr = findCorrespondingScheduler(xlinkFD); - ASSERT_XLINK(curr != NULL); - - xLinkEventPriv_t* event; - XLINK_RET_ERR_IF(pthread_mutex_lock(&(curr->queueMutex)) != 0, 1); - for (event = curr->lQueue.q; - event < curr->lQueue.q + MAX_EVENTS; - event++) - { - if (((event->packet.header.id == id || id == -1) - && event->packet.header.type == type - && event->packet.header.streamId == stream)) - { - mvLog(MVLOG_DEBUG,"served**************** %d %s\n", - (int)event->packet.header.id, - TypeToStr((int)event->packet.header.type)); - event->isServed = EVENT_SERVED; - XLINK_RET_ERR_IF(pthread_mutex_unlock(&(curr->queueMutex)) != 0, 1); - return 1; - } - } - XLINK_RET_ERR_IF(pthread_mutex_unlock(&(curr->queueMutex)) != 0, 1); - return 0; -} - // ------------------------------------ // XLinkDispatcher.h implementation. End. // ------------------------------------ @@ -645,10 +674,19 @@ static void* eventReader(void* ctx) DispatcherAddEvent(EVENT_REMOTE, &event); - if (event.header.type == XLINK_RESET_REQ) { - curr->resetXLink = 1; - mvLog(MVLOG_DEBUG,"Read XLINK_RESET_REQ, stopping eventReader thread."); +#ifdef __PC__ + // Stop receiving events when receive confirmation that the device acknowledged the reset request + if (event.header.type == XLINK_RESET_RESP) { + mvLog(MVLOG_DEBUG,"Read XLINK_RESET_RESP, stopping eventReader thread."); + break; } +#else + // Stop receiving events from remote when receive a XLINK_RESET_REQ + if (event.header.type == XLINK_RESET_REQ) { + mvLog(MVLOG_DEBUG,"Read XLINK_RESET_REQ, stopping eventReader thread."); + break; + } +#endif } return 0; @@ -751,13 +789,13 @@ static int isEventTypeRequest(xLinkEventPriv_t* event) static void postAndMarkEventServed(xLinkEventPriv_t *event) { - if (event->retEv){ + if (event->retEv) { // the xLinkEventPriv_t slot pointed by "event" will be // re-cycled as soon as we mark it as EVENT_SERVED, // so before that, we copy the result event into XLink API layer *(event->retEv) = event->packet; } - if(event->sem){ + if (event->sem && event->isServed != EVENT_DROPPED) { if (XLink_sem_post(event->sem)) { mvLog(MVLOG_ERROR,"can't post semaphore\n"); } @@ -811,7 +849,7 @@ static int dispatcherRequestServe(xLinkEventPriv_t * event, xLinkSchedulerState_ xLinkEventHeader_t *header = &event->packet.header; if (header->flags.bitField.block){ //block is requested event->isServed = EVENT_BLOCKED; - } else if(header->flags.bitField.localServe == 1 || + } else if (header->flags.bitField.localServe == 1 || (header->flags.bitField.ack == 0 && header->flags.bitField.nack == 1)){ //this event is served locally, or it is failed postAndMarkEventServed(event); @@ -820,7 +858,7 @@ static int dispatcherRequestServe(xLinkEventPriv_t * event, xLinkSchedulerState_ event->isServed = EVENT_PENDING; mvLog(MVLOG_DEBUG,"------------------------UNserved %s\n", TypeToStr(event->packet.header.type)); - }else{ + } else { return 1; } return 0; @@ -836,16 +874,31 @@ static int dispatcherResponseServe(xLinkEventPriv_t * event, xLinkSchedulerState xLinkEventHeader_t *header = &curr->lQueue.q[i].packet.header; xLinkEventHeader_t *evHeader = &event->packet.header; - if (curr->lQueue.q[i].isServed == EVENT_PENDING && - header->id == evHeader->id && - header->type == evHeader->type - XLINK_REQUEST_LAST -1) + if (header->id == evHeader->id && + header->type == evHeader->type - XLINK_REQUEST_LAST - 1) { - mvLog(MVLOG_DEBUG,"----------------------ISserved %s\n", - TypeToStr(header->type)); - //propagate back flags - header->flags = evHeader->flags; - postAndMarkEventServed(&curr->lQueue.q[i]); - break; + if (curr->lQueue.q[i].packet.header.type == XLINK_WRITE_REQ || + curr->lQueue.q[i].packet.header.type == XLINK_READ_REQ) { + while (!curr->lQueue.q[i].packet.header.canBeServed) continue; + } + if (curr->lQueue.q[i].isServed == EVENT_PENDING) { + mvLog(MVLOG_DEBUG,"----------------------ISserved %s\n", + TypeToStr(header->type)); + //propagate back flags + header->flags = evHeader->flags; + postAndMarkEventServed(&curr->lQueue.q[i]); + break; + } else if (curr->lQueue.q[i].isServed == EVENT_DROPPED) { + mvLog(MVLOG_DEBUG,"----------------------ISdropped %s\n", + TypeToStr(header->type)); + streamDesc_t* stream = getStreamById(event->packet.deviceHandle.xLinkFD, + event->packet.header.streamId); + if (stream->remoteFillLevel) + stream->remoteFillLevel -= event->packet.header.size; + releaseStream(stream); + postAndMarkEventServed(&curr->lQueue.q[i]); + break; + } } } if (i == MAX_EVENTS) { @@ -1149,6 +1202,13 @@ static XLinkError_t sendEvents(xLinkSchedulerState_t* curr) { XLINK_RET_ERR_IF(pthread_mutex_unlock(&(curr->queueMutex)) != 0, X_LINK_ERROR); mvLog(MVLOG_ERROR, "Event sending failed"); } +#ifndef __PC__ + // Stop scheduler thread after XLINK_RESET_RESP was successfully sent to host + if (toSend->header.type == XLINK_RESET_RESP) { + curr->resetXLink = 1; + mvLog(MVLOG_DEBUG, "Stop scheduler thread."); + } +#endif } else { XLINK_RET_ERR_IF(pthread_mutex_unlock(&(curr->queueMutex)) != 0, X_LINK_ERROR); } diff --git a/src/plugins/intel_myriad/third_party/XLink/shared/src/XLinkDispatcherImpl.c b/src/plugins/intel_myriad/third_party/XLink/shared/src/XLinkDispatcherImpl.c index e8133ba78f2..da8c98c99c5 100644 --- a/src/plugins/intel_myriad/third_party/XLink/shared/src/XLinkDispatcherImpl.c +++ b/src/plugins/intel_myriad/third_party/XLink/shared/src/XLinkDispatcherImpl.c @@ -169,7 +169,8 @@ int dispatcherLocalEventGetResponse(xLinkEvent_t* event, xLinkEvent_t* response) ASSERT_XLINK(stream); XLINK_EVENT_ACKNOWLEDGE(event); uint32_t releasedSize = 0; - releasePacketFromStream(stream, &releasedSize); + if (stream->blockedPackets) + releasePacketFromStream(stream, &releasedSize); event->header.size = releasedSize; releaseStream(stream); break; @@ -231,6 +232,11 @@ int dispatcherLocalEventGetResponse(xLinkEvent_t* event, xLinkEvent_t* response) mvLog(MVLOG_DEBUG,"XLINK_PING_REQ - do nothing\n"); break; } + case XLINK_DROP_REQ: + { + XLINK_EVENT_ACKNOWLEDGE(event); + break; + } case XLINK_WRITE_RESP: case XLINK_READ_RESP: case XLINK_READ_REL_RESP: @@ -243,6 +249,8 @@ int dispatcherLocalEventGetResponse(xLinkEvent_t* event, xLinkEvent_t* response) //should not happen event->header.flags.bitField.localServe = 1; break; + case XLINK_DROP_RESP: + break; default: { mvLog(MVLOG_ERROR, @@ -271,16 +279,22 @@ int dispatcherRemoteEventGetResponse(xLinkEvent_t* event, xLinkEvent_t* response response->header.size = event->header.size; response->header.streamId = event->header.streamId; response->deviceHandle = event->deviceHandle; + stream = getStreamById(event->deviceHandle.xLinkFD, + event->header.streamId); + ASSERT_XLINK(stream); XLINK_EVENT_ACKNOWLEDGE(response); - // we got some data. We should unblock a blocked read - int xxx = DispatcherUnblockEvent(-1, - XLINK_READ_REQ, - response->header.streamId, - event->deviceHandle.xLinkFD); - (void) xxx; - mvLog(MVLOG_DEBUG,"unblocked from stream %d %d\n", - (int)response->header.streamId, (int)xxx); + if (!event->header.dropped) + { + // we got some data. We should unblock a blocked read + int xxx = DispatcherUnblockEvent(-1, + XLINK_READ_REQ, + response->header.streamId, + event->deviceHandle.xLinkFD); + mvLog(MVLOG_DEBUG,"unblocked from stream %d %d\n", + (int)response->header.streamId, (int)xxx); + } + releaseStream(stream); } break; case XLINK_READ_REQ: @@ -288,6 +302,8 @@ int dispatcherRemoteEventGetResponse(xLinkEvent_t* event, xLinkEvent_t* response case XLINK_READ_REL_SPEC_REQ: XLINK_EVENT_ACKNOWLEDGE(response); response->header.type = XLINK_READ_REL_SPEC_RESP; + response->header.size = event->header.size; + response->header.streamId = event->header.streamId; response->deviceHandle = event->deviceHandle; stream = getStreamById(event->deviceHandle.xLinkFD, event->header.streamId); @@ -314,6 +330,8 @@ int dispatcherRemoteEventGetResponse(xLinkEvent_t* event, xLinkEvent_t* response case XLINK_READ_REL_REQ: XLINK_EVENT_ACKNOWLEDGE(response); response->header.type = XLINK_READ_REL_RESP; + response->header.size = event->header.size; + response->header.streamId = event->header.streamId; response->deviceHandle = event->deviceHandle; stream = getStreamById(event->deviceHandle.xLinkFD, event->header.streamId); @@ -422,6 +440,22 @@ int dispatcherRemoteEventGetResponse(xLinkEvent_t* event, xLinkEvent_t* response response->deviceHandle = event->deviceHandle; // need to send the response, serve the event and then reset break; + case XLINK_DROP_REQ: + { + response->header.type = XLINK_DROP_RESP; + response->header.size = event->header.size; + response->header.streamId = event->header.streamId; + response->deviceHandle = event->deviceHandle; + XLINK_EVENT_ACKNOWLEDGE(response); + + stream = getStreamById(event->deviceHandle.xLinkFD, + event->header.streamId); + uint32_t releasedSize = 0; + if (stream->blockedPackets) + releasePacketFromStream(stream, &releasedSize); + releaseStream(stream); + break; + } case XLINK_WRITE_RESP: break; case XLINK_READ_RESP: @@ -470,6 +504,8 @@ int dispatcherRemoteEventGetResponse(xLinkEvent_t* event, xLinkEvent_t* response break; case XLINK_RESET_RESP: break; + case XLINK_DROP_RESP: + break; default: { mvLog(MVLOG_ERROR, From fad66d8442a4d60f7fdcd2ae4fe78368ac252b7d Mon Sep 17 00:00:00 2001 From: Maxim Gordeev Date: Thu, 31 Mar 2022 10:09:31 +0300 Subject: [PATCH 19/39] [IE Samples] New command line parameters format for speech sample (#11051) * New command line parameters format for speech sample * fixed notes * changed format for scale factor * changed format for scale factor in tests * added more variants, when name is directy specified for i/o/r like it is done for sf * removed nthreads flag * fixed notes * changed output params * updated tests with new format Co-authored-by: Alexander Zhogov --- samples/cpp/speech_sample/main.cpp | 64 +++++++++++-------- samples/cpp/speech_sample/speech_sample.hpp | 7 -- samples/cpp/speech_sample/utils.hpp | 43 +++++++++++-- .../common/samples_common_test_clas.py | 14 ++-- .../smoke_tests/test_speech_sample.py | 25 +++++++- 5 files changed, 105 insertions(+), 48 deletions(-) diff --git a/samples/cpp/speech_sample/main.cpp b/samples/cpp/speech_sample/main.cpp index 861058bf94c..c034c64dad7 100644 --- a/samples/cpp/speech_sample/main.cpp +++ b/samples/cpp/speech_sample/main.cpp @@ -49,7 +49,10 @@ int main(int argc, char* argv[]) { BaseFile* fileOutput; ArkFile arkFile; NumpyFile numpyFile; - auto extInputFile = fileExt(FLAGS_i); + std::pair> input_data; + if (!FLAGS_i.empty()) + input_data = parse_parameters(FLAGS_i); + auto extInputFile = fileExt(input_data.first); if (extInputFile == "ark") { file = &arkFile; } else if (extInputFile == "npz") { @@ -60,9 +63,9 @@ int main(int argc, char* argv[]) { std::vector inputFiles; std::vector numBytesThisUtterance; uint32_t numUtterances(0); - if (!FLAGS_i.empty()) { + if (!input_data.first.empty()) { std::string outStr; - std::istringstream stream(FLAGS_i); + std::istringstream stream(input_data.first); uint32_t currentNumUtterances(0), currentNumBytesThisUtterance(0); while (getline(stream, outStr, ',')) { std::string filename(fileNameNoExt(outStr) + "." + extInputFile); @@ -89,19 +92,26 @@ int main(int argc, char* argv[]) { std::vector output_names; std::vector ports; // --------------------------- Processing custom outputs --------------------------------------------- - if (!FLAGS_oname.empty()) { - output_names = convert_str_to_vector(FLAGS_oname); - for (const auto& output_name : output_names) { - auto pos_layer = output_name.rfind(":"); - if (pos_layer == std::string::npos) { - throw std::logic_error("Output " + output_name + " doesn't have a port"); - } - outputs.push_back(output_name.substr(0, pos_layer)); - try { - ports.push_back(std::stoi(output_name.substr(pos_layer + 1))); - } catch (const std::exception&) { - throw std::logic_error("Ports should have integer type"); - } + std::pair> output_data; + std::pair> reference_data; + if (!FLAGS_o.empty()) + output_data = parse_parameters(FLAGS_o); + if (!FLAGS_r.empty()) + reference_data = parse_parameters(FLAGS_r); + if (!output_data.second.empty()) + output_names = output_data.second; + else if (!reference_data.second.empty()) + output_names = reference_data.second; + for (const auto& output_name : output_names) { + auto pos_layer = output_name.rfind(":"); + if (pos_layer == std::string::npos) { + throw std::logic_error("Output " + output_name + " doesn't have a port"); + } + outputs.push_back(output_name.substr(0, pos_layer)); + try { + ports.push_back(std::stoi(output_name.substr(pos_layer + 1))); + } catch (const std::exception&) { + throw std::logic_error("Ports should have integer type"); } } // ------------------------------ Preprocessing ------------------------------------------------------ @@ -304,8 +314,8 @@ int main(int argc, char* argv[]) { std::vector ptrInputBlobs; auto cInputInfo = executableNet.inputs(); check_number_of_inputs(cInputInfo.size(), numInputFiles); - if (!FLAGS_iname.empty()) { - std::vector inputNameBlobs = convert_str_to_vector(FLAGS_iname); + if (!input_data.second.empty()) { + std::vector inputNameBlobs = input_data.second; if (inputNameBlobs.size() != cInputInfo.size()) { std::string errMessage(std::string("Number of network inputs ( ") + std::to_string(cInputInfo.size()) + " ) is not equal to the number of inputs entered in the -iname argument ( " + @@ -328,15 +338,15 @@ int main(int argc, char* argv[]) { std::vector output_name_files; std::vector reference_name_files; size_t count_file = 1; - if (!FLAGS_o.empty()) { - output_name_files = convert_str_to_vector(FLAGS_o); + if (!output_data.first.empty()) { + output_name_files = convert_str_to_vector(output_data.first); if (output_name_files.size() != outputs.size() && !outputs.empty()) { throw std::logic_error("The number of output files is not equal to the number of network outputs."); } count_file = output_name_files.empty() ? 1 : output_name_files.size(); } - if (!FLAGS_r.empty()) { - reference_name_files = convert_str_to_vector(FLAGS_r); + if (!reference_data.first.empty()) { + reference_name_files = convert_str_to_vector(reference_data.first); if (reference_name_files.size() != outputs.size() && !outputs.empty()) { throw std::logic_error("The number of reference files is not equal to the number of network outputs."); } @@ -429,9 +439,9 @@ int main(int argc, char* argv[]) { BaseFile* fileReferenceScores; std::string refUtteranceName; - if (!FLAGS_r.empty()) { + if (!reference_data.first.empty()) { /** Read file with reference scores **/ - auto exReferenceScoresFile = fileExt(FLAGS_r); + auto exReferenceScoresFile = fileExt(reference_data.first); if (exReferenceScoresFile == "ark") { fileReferenceScores = &arkFile; } else if (exReferenceScoresFile == "npz") { @@ -540,12 +550,12 @@ int main(int argc, char* argv[]) { continue; } ptrInputBlobs.clear(); - if (FLAGS_iname.empty()) { + if (input_data.second.empty()) { for (auto& input : cInputInfo) { ptrInputBlobs.push_back(inferRequest.inferRequest.get_tensor(input)); } } else { - std::vector inputNameBlobs = convert_str_to_vector(FLAGS_iname); + std::vector inputNameBlobs = input_data.second; for (const auto& input : inputNameBlobs) { ov::Tensor blob = inferRequests.begin()->inferRequest.get_tensor(input); if (!blob) { @@ -638,7 +648,7 @@ int main(int argc, char* argv[]) { for (size_t next_output = 0; next_output < count_file; next_output++) { if (!FLAGS_o.empty()) { - auto exOutputScoresFile = fileExt(FLAGS_o); + auto exOutputScoresFile = fileExt(output_data.first); if (exOutputScoresFile == "ark") { fileOutput = &arkFile; } else if (exOutputScoresFile == "npz") { diff --git a/samples/cpp/speech_sample/speech_sample.hpp b/samples/cpp/speech_sample/speech_sample.hpp index f398d709cb3..c0961b768ac 100644 --- a/samples/cpp/speech_sample/speech_sample.hpp +++ b/samples/cpp/speech_sample/speech_sample.hpp @@ -96,10 +96,6 @@ static const char scale_factor_message[] = /// @brief message for batch size argument static const char batch_size_message[] = "Optional. Batch size 1-8 (default 1)"; -/// @brief message for #threads for CPU inference -static const char infer_num_threads_message[] = "Optional. Number of threads to use for concurrent async" - " inference requests on the GNA."; - /// @brief message for left context window argument static const char context_window_message_l[] = "Optional. Number of frames for left context windows (default is 0). " @@ -184,9 +180,6 @@ DEFINE_string(sf, "", scale_factor_message); /// @brief Batch size (default 0) DEFINE_int32(bs, 0, batch_size_message); -/// @brief Number of threads to use for inference on the CPU (also affects Hetero cases) -DEFINE_int32(nthreads, 1, infer_num_threads_message); - /// @brief Right context window size (default 0) DEFINE_int32(cw_r, 0, context_window_message_r); diff --git a/samples/cpp/speech_sample/utils.hpp b/samples/cpp/speech_sample/utils.hpp index 4626cf55207..f194c4e66ab 100644 --- a/samples/cpp/speech_sample/utils.hpp +++ b/samples/cpp/speech_sample/utils.hpp @@ -432,7 +432,7 @@ bool check_name(const ov::OutputVector& nodes, const std::string& node_name) { /** * @brief Parse scale factors per input - * Format : :,: or just + * Format : =,= or just * @param inputs model inputs * @param values_string values_string input string * @return map of scale factors per input @@ -454,11 +454,11 @@ std::map parse_scale_factors(const ov::OutputVector& inputs, std::map result; auto scale_factor_strings = split(values_string, ','); for (auto& scale_factor_string : scale_factor_strings) { - auto values = split(scale_factor_string, ':'); + auto values = split(scale_factor_string, '='); if (values.size() == 1) { if (scale_factor_strings.size() != 1) { throw std::logic_error("Unrecognized scale factor format! " - "Please specify :,: or " + "Please specify =,= or " "just to be applied to all inputs"); } auto scale_factor = get_sf(values.at(0)); @@ -468,8 +468,7 @@ std::map parse_scale_factors(const ov::OutputVector& inputs, } else if (values.size() > 0) { auto sf_sting = values.back(); values.pop_back(); - // input name can contain port, concat back - auto input_name = concat(values, ':'); + auto input_name = values.back(); check_name(inputs, input_name); result[input_name] = get_sf(sf_sting, input_name); } @@ -535,3 +534,37 @@ std::map parse_input_layouts(const std::string& layout throw std::logic_error("Can't parse input parameter string: " + layout_string); return return_value; } + +/** + * @brief Parse parameters for inputs/outputs like as "=,=" or + * "" in case of one input/output + * @param file_paths_string input/output path + * @return pair of filename and vector of tensor_names + */ +std::pair> parse_parameters(const std::string file_paths_string) { + auto search_string = file_paths_string; + char comma_delim = ','; + char equal_delim = '='; + std::string filename = ""; + std::vector tensor_names; + std::vector filenames; + if (!std::count(search_string.begin(), search_string.end(), comma_delim) && + !std::count(search_string.begin(), search_string.end(), equal_delim)) { + return {search_string, tensor_names}; + } + search_string += comma_delim; + std::vector splitted = split(search_string, comma_delim); + for (size_t j = 0; j < splitted.size(); j++) { + auto semicolon_pos = splitted[j].find_first_of(equal_delim); + if (semicolon_pos != std::string::npos) { + tensor_names.push_back(splitted[j].substr(0, semicolon_pos)); + filenames.push_back(splitted[j].substr(semicolon_pos + 1, std::string::npos)); + } + } + for (std::vector::const_iterator name = filenames.begin(); name != filenames.end(); ++name) { + filename += *name; + if (name != filenames.end() - 1) + filename += comma_delim; + } + return {filename, tensor_names}; +} \ No newline at end of file diff --git a/tests/samples_tests/smoke_tests/common/samples_common_test_clas.py b/tests/samples_tests/smoke_tests/common/samples_common_test_clas.py index 9e789976a54..b62c2fdb6f8 100644 --- a/tests/samples_tests/smoke_tests/common/samples_common_test_clas.py +++ b/tests/samples_tests/smoke_tests/common/samples_common_test_clas.py @@ -186,16 +186,16 @@ class SamplesCommonTestClass(): return model @staticmethod - def join_env_path(param, executable_path): + def join_env_path(param, executable_path, complete_path=True): gpu_lib_path = os.path.join(os.environ.get('IE_APP_PATH'), 'lib') if 'i' in param: # If batch > 1, then concatenate images if ' ' in param['i']: param['i'] = param['i'].split(' ') - else: + elif complete_path: param['i'] = list([param['i']]) for k in param.keys(): - if ('i' == k): + if ('i' == k) and complete_path: param['i'] = [os.path.join(Environment.env['test_data'], e) for e in param['i']] param['i'] = ' '.join(param['i']) elif ('ref_m' == k): @@ -236,10 +236,10 @@ class SamplesCommonTestClass(): param['l'] = os.path.join(Environment.env['test_data'], param['l']) elif ('pp' == k): param['pp'] = gpu_lib_path - elif ('r' == k): + elif ('r' == k) and complete_path: if len(param['r']) > 0: param['r'] = os.path.join(Environment.env['test_data'], param['r']) - elif ('o' == k): + elif ('o' == k) and complete_path: param['o'] = os.path.join(Environment.env['out_directory'], param['o']) elif ('wg' == k): param['wg'] = os.path.join(Environment.env['out_directory'], param['wg']) @@ -344,7 +344,7 @@ class SamplesCommonTestClass(): "Path for test data {} is not exist!".format(Environment.env['test_data']) cls.output_dir = Environment.env['out_directory'] - def _test(self, param, use_preffix=True, get_cmd_func=None, get_shell_result=False, long_hyphen=None): + def _test(self, param, use_preffix=True, get_cmd_func=None, get_shell_result=False, long_hyphen=None, complete_path=True): """ :param param: :param use_preffix: use it when sample doesn't require keys (i.e. hello_classification @@ -379,7 +379,7 @@ class SamplesCommonTestClass(): if get_cmd_func is None: get_cmd_func = self.get_cmd_line - self.join_env_path(param_cp, executable_path=self.executable_path) + self.join_env_path(param_cp, executable_path=self.executable_path, complete_path=complete_path) # Updating all attributes in the original dictionary (param), because param_cp was changes (join_env_path) for key in param.keys(): diff --git a/tests/samples_tests/smoke_tests/test_speech_sample.py b/tests/samples_tests/smoke_tests/test_speech_sample.py index 0ee5095d719..686a1e803a8 100644 --- a/tests/samples_tests/smoke_tests/test_speech_sample.py +++ b/tests/samples_tests/smoke_tests/test_speech_sample.py @@ -15,6 +15,7 @@ import pytest import sys import logging as log from common.samples_common_test_clas import SamplesCommonTestClass +from common.samples_common_test_clas import Environment from common.samples_common_test_clas import get_tests from common.common_utils import parse_avg_err @@ -27,11 +28,24 @@ test_data = get_tests(cmd_params={'i': [os.path.join('ark', 'dev93_10.ark')], 'o': ['res_output.ark'], 'r': [os.path.join('ark', 'dev93_scores_10.ark')], 'qb': [8, 16], - 'sf': ["Parameter:2175.43", "2175.43"], + 'sf': ["2175.43"], 'q': ["static", "user"], 'd': ['GNA_SW_EXACT']}, use_device=False ) + +new_format_test_data = get_tests(cmd_params={'i': ['Parameter=' + os.path.join(Environment.env['test_data'], 'ark', 'dev93_10.ark')], + 'm': [os.path.join('wsj', 'FP32', 'wsj_dnn5b.xml')], + 'layout': ["[NC]"], + 'bs': [1], + 'o': ['affinetransform14/Fused_Add_:0=' + os.path.join(Environment.env['test_data'], 'res_output.ark')], + 'r': ['affinetransform14/Fused_Add_:0=' + os.path.join(Environment.env['test_data'], 'ark', 'dev93_scores_10.ark')], + 'qb': [8], + 'sf': ["Parameter=2175.43"], + 'q': ["static"], + 'd': ['GNA_SW_EXACT']}, + use_device=False + ) class TestSpeechSample(SamplesCommonTestClass): @classmethod @@ -43,7 +57,14 @@ class TestSpeechSample(SamplesCommonTestClass): @pytest.mark.parametrize("param", test_data) def test_speech_sample_nthreads(self, param): stdout = self._test(param).split('\n') - assert os.path.isfile(param['o']), "Ark file after infer was not found" + + avg_error = parse_avg_err(stdout) + log.info('Average scores diff: {}'.format(avg_error)) + assert avg_error <= self.threshold + + @pytest.mark.parametrize("param", new_format_test_data) + def test_speech_sample_new_format(self, param): + stdout = self._test(param, complete_path=False).split('\n') avg_error = parse_avg_err(stdout) log.info('Average scores diff: {}'.format(avg_error)) From c795382b1f14392fa4a8809a654e464a426583d7 Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Thu, 31 Mar 2022 10:49:40 +0300 Subject: [PATCH 20/39] Configurable OpenCL usage in BA (#11344) --- samples/cpp/benchmark_app/CMakeLists.txt | 71 +++++++++++++----------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/samples/cpp/benchmark_app/CMakeLists.txt b/samples/cpp/benchmark_app/CMakeLists.txt index 39f7bf890e2..666b65929da 100644 --- a/samples/cpp/benchmark_app/CMakeLists.txt +++ b/samples/cpp/benchmark_app/CMakeLists.txt @@ -14,39 +14,48 @@ ie_add_sample(NAME ${TARGET_NAME} # Optional OpenCL dependnency -find_package(OpenCL) - -find_path(OpenCL_HPP_INCLUDE_DIR - NAMES - CL/cl2.hpp OpenCL/cl2.hpp - HINTS - ${opencl_root_hints} - ENV "PROGRAMFILES(X86)" - ENV AMDAPPSDKROOT - ENV INTELOCLSDKROOT - ENV NVSDKCOMPUTE_ROOT - ENV CUDA_PATH - ENV ATISTREAMSDKROOT - ENV OCL_ROOT - PATH_SUFFIXES - include - OpenCL/common/inc - "AMD APP/include") - -if(TARGET OpenCL) - # Use OpenCL CPP headers from sources if present - set(OpenCL_HEADERS ${OPENCL_HEADERS_DIR}) - set(OpenCL_LIB "OpenCL") -elseif(OpenCL_HPP_INCLUDE_DIR) - # Append OpenCL CPP headers to C headers and use both - set(OpenCL_HEADERS ${OpenCL_INCLUDE_DIR} ${OpenCL_HPP_INCLUDE_DIR}) - set(OpenCL_LIB "OpenCL::OpenCL") +if(DEFINED ENABLE_INTEL_GPU AND NOT ENABLE_INTEL_GPU) + # Intel GPU plugin is turned off explicitly + option(SAMPLES_ENABLE_OPENCL "Use OpenCL in benchmark_app" OFF) +else() + option(SAMPLES_ENABLE_OPENCL "Use OpenCL in benchmark_app" ON) endif() -if(OpenCL_FOUND AND OpenCL_HEADERS) - target_link_libraries(${TARGET_NAME} PRIVATE ${OpenCL_LIB}) - target_include_directories(${TARGET_NAME} PRIVATE ${OpenCL_HEADERS}) - target_compile_definitions(${TARGET_NAME} PRIVATE HAVE_GPU_DEVICE_MEM_SUPPORT) +if(SAMPLES_ENABLE_OPENCL) + find_package(OpenCL) + + find_path(OpenCL_HPP_INCLUDE_DIR + NAMES + CL/cl2.hpp OpenCL/cl2.hpp + HINTS + ${opencl_root_hints} + ENV "PROGRAMFILES(X86)" + ENV AMDAPPSDKROOT + ENV INTELOCLSDKROOT + ENV NVSDKCOMPUTE_ROOT + ENV CUDA_PATH + ENV ATISTREAMSDKROOT + ENV OCL_ROOT + PATH_SUFFIXES + include + OpenCL/common/inc + "AMD APP/include") + + if(TARGET OpenCL) + # Use OpenCL CPP headers from sources if present + set(OpenCL_HEADERS ${OPENCL_HEADERS_DIR}) + set(OpenCL_LIB "OpenCL") + elseif(OpenCL_HPP_INCLUDE_DIR) + # Append OpenCL CPP headers to C headers and use both + set(OpenCL_HEADERS ${OpenCL_INCLUDE_DIR} ${OpenCL_HPP_INCLUDE_DIR}) + set(OpenCL_LIB "OpenCL::OpenCL") + endif() + + if(OpenCL_FOUND AND OpenCL_HEADERS) + target_link_libraries(${TARGET_NAME} PRIVATE ${OpenCL_LIB}) + target_include_directories(${TARGET_NAME} PRIVATE ${OpenCL_HEADERS}) + target_compile_definitions(${TARGET_NAME} PRIVATE HAVE_GPU_DEVICE_MEM_SUPPORT) + endif() endif() # Optional OpenCV dependency From 7d0750fa2acc6d3b6cd2f456b8b1cdac34e4704a Mon Sep 17 00:00:00 2001 From: Alina Kladieva Date: Thu, 31 Mar 2022 11:12:19 +0300 Subject: [PATCH 21/39] Add SKIP_IF_CURRENT_TEST_IS_DISABLED macros for needed cases (#11335) --- .../gpu/remote_blob_tests/cldnn_remote_blob_tests.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/tests/functional/plugin/gpu/remote_blob_tests/cldnn_remote_blob_tests.cpp b/src/tests/functional/plugin/gpu/remote_blob_tests/cldnn_remote_blob_tests.cpp index 3ff0f73c03a..7e953a01dff 100644 --- a/src/tests/functional/plugin/gpu/remote_blob_tests/cldnn_remote_blob_tests.cpp +++ b/src/tests/functional/plugin/gpu/remote_blob_tests/cldnn_remote_blob_tests.cpp @@ -265,6 +265,9 @@ TEST_P(RemoteBlob_Test, smoke_canInferOnUserQueue_out_of_order) { #if defined _WIN32 GTEST_SKIP(); #endif + + SKIP_IF_CURRENT_TEST_IS_DISABLED() + CNNNetwork net(fn_ptr); net.getInputsInfo().begin()->second->setLayout(Layout::NCHW); @@ -360,6 +363,9 @@ TEST_P(RemoteBlob_Test, smoke_canInferOnUserQueue_in_order) { #if defined _WIN32 GTEST_SKIP(); #endif + + SKIP_IF_CURRENT_TEST_IS_DISABLED() + CNNNetwork net(fn_ptr); net.getInputsInfo().begin()->second->setLayout(Layout::NCHW); From f45ca99de657d87d49f80b1d2068bec1bfcca43b Mon Sep 17 00:00:00 2001 From: Przemyslaw Wysocki Date: Thu, 31 Mar 2022 11:14:51 +0200 Subject: [PATCH 22/39] [PYTHON] Add ov::clone_function bindings (#11331) * Add clone_function bindings * Add clone_function binding tests * Debugging changes * Minor changes * Code style * Add an assert * Update src/bindings/python/tests/test_ngraph/test_basic.py * Update src/bindings/python/src/pyopenvino/graph/model.cpp Co-authored-by: Jan Iwaszkiewicz Co-authored-by: Anastasia Kuporosova --- .../src/openvino/runtime/utils/__init__.py | 1 + .../python/src/pyopenvino/graph/model.cpp | 12 ++++++++++ .../python/src/pyopenvino/graph/util.cpp | 14 +++++++++++ .../python/tests/test_ngraph/test_basic.py | 24 +++++++++++++++++++ 4 files changed, 51 insertions(+) diff --git a/src/bindings/python/src/openvino/runtime/utils/__init__.py b/src/bindings/python/src/openvino/runtime/utils/__init__.py index 90356e3bd98..b2432405f85 100644 --- a/src/bindings/python/src/openvino/runtime/utils/__init__.py +++ b/src/bindings/python/src/openvino/runtime/utils/__init__.py @@ -4,4 +4,5 @@ """Generic utilities. Factor related functions out to separate files.""" from openvino.pyopenvino.util import numpy_to_c +from openvino.pyopenvino.util import clone_model from openvino.pyopenvino.util import get_constant_from_source, replace_node, replace_output_update_name diff --git a/src/bindings/python/src/pyopenvino/graph/model.cpp b/src/bindings/python/src/pyopenvino/graph/model.cpp index 5bc4b6e4dcc..ab76a678623 100644 --- a/src/bindings/python/src/pyopenvino/graph/model.cpp +++ b/src/bindings/python/src/pyopenvino/graph/model.cpp @@ -7,6 +7,7 @@ #include #include +#include "openvino/core/graph_util.hpp" #include "openvino/core/model.hpp" // ov::Model #include "openvino/core/partial_shape.hpp" #include "openvino/op/parameter.hpp" // ov::op::v0::Parameter @@ -656,6 +657,17 @@ void regclass_graph_Model(py::module m) { :rtype: bool )"); + model.def( + "clone", + [](ov::Model& self) { + return ov::clone_model(self); + }, + R"( + Return a copy of self. + :return: A copy of self. + :rtype: openvino.runtime.Model + )"); + model.def("__repr__", [](const ov::Model& self) { std::string class_name = py::cast(self).get_type().attr("__name__").cast(); diff --git a/src/bindings/python/src/pyopenvino/graph/util.cpp b/src/bindings/python/src/pyopenvino/graph/util.cpp index e9d01b32760..4393e64456c 100644 --- a/src/bindings/python/src/pyopenvino/graph/util.cpp +++ b/src/bindings/python/src/pyopenvino/graph/util.cpp @@ -36,6 +36,20 @@ void regmodule_graph_util(py::module m) { from the resulting bound, otherwise Null. :rtype: openvino.runtime.op.Constant or openvino.runtime.Node )"); + mod.def( + "clone_model", + [](ov::Model& model) { + return ov::clone_model(model); + }, + py::arg("model"), + R"( + Creates a copy of a model object. + + :param model: Model to copy. + :type model: openvino.runtime.Model + :return: A copy of Model. + :rtype: openvino.runtime.Model + )"); mod.def("replace_output_update_name", &ov::replace_output_update_name, py::arg("output"), py::arg("target_output")); diff --git a/src/bindings/python/tests/test_ngraph/test_basic.py b/src/bindings/python/tests/test_ngraph/test_basic.py index a355bb80bee..da0f39531ba 100644 --- a/src/bindings/python/tests/test_ngraph/test_basic.py +++ b/src/bindings/python/tests/test_ngraph/test_basic.py @@ -327,6 +327,30 @@ def test_set_argument(): assert np.allclose(data1 + data2, output) +def test_clone_model(): + # Create an original model + shape = [2, 2] + parameter_a = ops.parameter(shape, dtype=np.float32, name="A") + parameter_b = ops.parameter(shape, dtype=np.float32, name="B") + model_original = ov.Model(parameter_a + parameter_b, [parameter_a, parameter_b]) + + # Make copies of it + model_copy1 = ov.utils.clone_model(model_original) + model_copy2 = model_original.clone() + + # Make changes to the copied models' inputs + model_copy1.reshape({"A": [3, 3], "B": [3, 3]}) + model_copy2.reshape({"A": [3, 3], "B": [3, 3]}) + + original_model_shapes = [single_input.get_shape() for single_input in model_original.inputs] + model_copy1_shapes = [single_input.get_shape() for single_input in model_copy1.inputs] + model_copy2_shapes = [single_input.get_shape() for single_input in model_copy2.inputs] + + assert original_model_shapes != model_copy1_shapes + assert original_model_shapes != model_copy2_shapes + assert model_copy1_shapes == model_copy2_shapes + + def test_result(): node = np.array([[11, 10], [1, 8], [3, 4]], dtype=np.float32) result = run_op_node([node], ops.result) From 23476c8eee30c3ad79b6b0085e7f6dac106dc979 Mon Sep 17 00:00:00 2001 From: Egor Shulman Date: Thu, 31 Mar 2022 12:35:27 +0300 Subject: [PATCH 23/39] CC for LPT transformation call in CPU plugin (#11341) * Use CC in LPT * Applied comment --- .../CMakeLists.txt | 2 + .../include/itt.hpp | 45 +++++++++++++++++++ .../low_precision_transformations/src/add.cpp | 4 +- .../src/align_quantization_intervals.cpp | 2 + .../src/align_quantization_parameters.cpp | 2 + .../src/assign_and_read_value.cpp | 4 +- .../src/avg_pool.cpp | 4 +- .../src/clamp.cpp | 4 +- .../src/concat.cpp | 4 +- .../src/convert.cpp | 4 +- .../src/convert_subtract_constant.cpp | 4 +- .../src/convolution.cpp | 4 +- .../src/convolution_backprop_data.cpp | 4 +- .../src/depth_to_space.cpp | 4 +- .../src/fake_quantize.cpp | 4 +- .../src/fake_quantize_decomposition.cpp | 4 +- .../src/fold_convert.cpp | 4 +- .../src/fold_fake_quantize.cpp | 4 +- .../src/fuse_convert.cpp | 4 +- .../src/fuse_multiply_to_fake_quantize.cpp | 4 +- .../src/fuse_subtract_to_fake_quantize.cpp | 4 +- .../src/group_convolution.cpp | 4 +- .../src/interpolate.cpp | 4 +- .../src/layer_transformation.cpp | 4 +- .../src/low_precision.cpp | 7 ++- .../markup_avg_pool_precision_preserved.cpp | 2 + .../src/markup_can_be_quantized.cpp | 2 + .../src/markup_per_tensor_quantization.cpp | 2 + .../src/markup_precisions.cpp | 2 + .../src/mat_mul.cpp | 4 +- .../src/max_pool.cpp | 4 +- .../src/move_fake_quantize.cpp | 4 +- .../src/multiply.cpp | 4 +- .../src/multiply_to_group_convolution.cpp | 4 +- .../low_precision_transformations/src/mvn.cpp | 4 +- .../src/normalize_l2.cpp | 4 +- .../low_precision_transformations/src/pad.cpp | 4 +- .../src/prelu.cpp | 4 +- .../src/propagate_precisions.cpp | 2 + .../pull_reshape_through_dequantization.cpp | 4 +- .../pull_transpose_through_dequantization.cpp | 4 +- .../src/reduce_max.cpp | 4 +- .../src/reduce_mean.cpp | 4 +- .../src/reduce_min.cpp | 4 +- .../src/reduce_sum.cpp | 4 +- .../src/relu.cpp | 4 +- .../src/reshape.cpp | 4 +- .../src/shuffle_channels.cpp | 4 +- .../src/split.cpp | 4 +- .../src/squeeze.cpp | 4 +- .../src/strided_slice.cpp | 4 +- .../src/subtract.cpp | 4 +- .../src/transpose.cpp | 4 +- .../src/unsqueeze.cpp | 4 +- .../src/variadic_split.cpp | 4 +- src/plugins/intel_cpu/src/itt.h | 13 ++++++ src/plugins/intel_cpu/src/plugin.cpp | 5 +++ 57 files changed, 220 insertions(+), 46 deletions(-) create mode 100644 src/common/low_precision_transformations/include/itt.hpp diff --git a/src/common/low_precision_transformations/CMakeLists.txt b/src/common/low_precision_transformations/CMakeLists.txt index 8e534fd9b15..79390ab6be3 100644 --- a/src/common/low_precision_transformations/CMakeLists.txt +++ b/src/common/low_precision_transformations/CMakeLists.txt @@ -33,6 +33,8 @@ target_include_directories(${TARGET_NAME}_obj PRIVATE $ +#include +#include + +namespace ngraph { +namespace pass { +namespace low_precision { +namespace itt { +namespace domains { + OV_ITT_DOMAIN(LowPrecisionTransformations); +} // namespace domains +} // namespace itt +} // namespace low_precision +} // namespace pass +} // namespace ngraph + +/* + * RUN_ON_FUNCTION_SCOPE macro allows to disable the run_on_function pass + * MATCHER_SCOPE macro allows to disable the MatcherPass if matcher isn't applied + * INTERNAL_OP_SCOPE macro allows to disable parts of internal nGraph operations if they are not used + */ +#if defined(SELECTIVE_BUILD_ANALYZER) + +#define INTERNAL_OP_SCOPE(region) OV_SCOPE(internal_op, region) + +#elif defined(SELECTIVE_BUILD) + +#define INTERNAL_OP_SCOPE(region) MATCHER_SCOPE_(internal_op, region) + +#else + +#define INTERNAL_OP_SCOPE(region) + +#endif diff --git a/src/common/low_precision_transformations/src/add.cpp b/src/common/low_precision_transformations/src/add.cpp index 498d9a0cbab..950bd3578b0 100644 --- a/src/common/low_precision_transformations/src/add.cpp +++ b/src/common/low_precision_transformations/src/add.cpp @@ -15,6 +15,7 @@ #include "low_precision/common/ie_lpt_exception.hpp" #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { @@ -93,6 +94,7 @@ std::shared_ptr fuseWithSubtract(const std::shared_ptr& } // namespace AddTransformation::AddTransformation(const Params& params) : EltwiseBaseTransformation(params) { + MATCHER_SCOPE(AddTransformation); auto matcher = ngraph::pattern::wrap_type(); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -103,7 +105,7 @@ AddTransformation::AddTransformation(const Params& params) : EltwiseBaseTransfor return transform(*context, m); }; - auto m = std::make_shared(matcher, "AddTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/align_quantization_intervals.cpp b/src/common/low_precision_transformations/src/align_quantization_intervals.cpp index bccf3f153bd..0b8061b69d1 100644 --- a/src/common/low_precision_transformations/src/align_quantization_intervals.cpp +++ b/src/common/low_precision_transformations/src/align_quantization_intervals.cpp @@ -9,6 +9,7 @@ #include "low_precision/propagate_through_precision_preserved.hpp" #include "low_precision/rt_info/intervals_alignment_attribute.hpp" #include "low_precision/rt_info/attribute_parameters.hpp" +#include "itt.hpp" using namespace ngraph; using namespace ngraph::pass::low_precision; @@ -17,6 +18,7 @@ ngraph::pass::low_precision::AlignQuantizationIntervals::AlignQuantizationInterv : defaultPrecisions(defaultPrecisions) {} bool ngraph::pass::low_precision::AlignQuantizationIntervals::run_on_model(const std::shared_ptr& f) { + RUN_ON_FUNCTION_SCOPE(AlignQuantizationIntervals); ngraph::pass::Manager manager; manager.set_per_pass_validation(false); std::shared_ptr intervalsAlignment = manager.register_pass(); diff --git a/src/common/low_precision_transformations/src/align_quantization_parameters.cpp b/src/common/low_precision_transformations/src/align_quantization_parameters.cpp index d273bbad832..b7feadef8d7 100644 --- a/src/common/low_precision_transformations/src/align_quantization_parameters.cpp +++ b/src/common/low_precision_transformations/src/align_quantization_parameters.cpp @@ -10,6 +10,7 @@ #include "low_precision/rt_info/per_tensor_quantization_attribute.hpp" #include "low_precision/update_shared_precision_preserved.hpp" #include "low_precision/rt_info/attribute_parameters.hpp" +#include "itt.hpp" using namespace ngraph; using namespace ngraph::pass::low_precision; @@ -18,6 +19,7 @@ ngraph::pass::low_precision::AlignQuantizationParameters::AlignQuantizationParam : defaultPrecisions(defaultPrecisions) {} bool ngraph::pass::low_precision::AlignQuantizationParameters::run_on_model(const std::shared_ptr& f) { + RUN_ON_FUNCTION_SCOPE(AlignQuantizationParameters); ngraph::pass::Manager manager; manager.set_per_pass_validation(false); std::shared_ptr propagation = manager.register_pass(); diff --git a/src/common/low_precision_transformations/src/assign_and_read_value.cpp b/src/common/low_precision_transformations/src/assign_and_read_value.cpp index d15c8edf961..c8d705f729b 100644 --- a/src/common/low_precision_transformations/src/assign_and_read_value.cpp +++ b/src/common/low_precision_transformations/src/assign_and_read_value.cpp @@ -11,6 +11,7 @@ #include #include #include "low_precision/fake_quantize.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { @@ -18,6 +19,7 @@ namespace low_precision { AssignAndReadValueTransformation::AssignAndReadValueTransformation(const std::shared_ptr function, const Params& params) : LayerTransformation(params), function(function) { + MATCHER_SCOPE(AssignAndReadValueTransformation); auto assign3 = pattern::wrap_type({ pattern::wrap_type() }); auto assign6 = pattern::wrap_type({ pattern::wrap_type() }); @@ -42,7 +44,7 @@ AssignAndReadValueTransformation::AssignAndReadValueTransformation(const std::sh auto m = std::make_shared( std::make_shared(OutputVector{ assign3, assign6 }), - "AssignAndReadValueTransformation"); + matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/avg_pool.cpp b/src/common/low_precision_transformations/src/avg_pool.cpp index a4724686c61..46af481db20 100644 --- a/src/common/low_precision_transformations/src/avg_pool.cpp +++ b/src/common/low_precision_transformations/src/avg_pool.cpp @@ -11,12 +11,14 @@ #include "low_precision/network_helper.hpp" #include "low_precision/rt_info/precision_preserved_attribute.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { AvgPoolTransformation::AvgPoolTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(AvgPoolTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -27,7 +29,7 @@ AvgPoolTransformation::AvgPoolTransformation(const Params& params) : LayerTransf return transform(*context, m); }; - auto m = std::make_shared(matcher, "AvgPoolTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/clamp.cpp b/src/common/low_precision_transformations/src/clamp.cpp index 528c67a2088..a7e3a717a07 100644 --- a/src/common/low_precision_transformations/src/clamp.cpp +++ b/src/common/low_precision_transformations/src/clamp.cpp @@ -9,12 +9,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { ClampTransformation::ClampTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(ClampTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -25,7 +27,7 @@ ClampTransformation::ClampTransformation(const Params& params) : LayerTransforma return transform(*context, m); }; - auto m = std::make_shared(matcher, "ClampTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/concat.cpp b/src/common/low_precision_transformations/src/concat.cpp index 383fa0fa0af..71f0d5db021 100644 --- a/src/common/low_precision_transformations/src/concat.cpp +++ b/src/common/low_precision_transformations/src/concat.cpp @@ -15,12 +15,14 @@ #include "low_precision/common/fake_quantize_dequantization.hpp" #include "low_precision/common/ie_lpt_exception.hpp" #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { ConcatTransformation::ConcatTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(ConcatTransformation); auto matcher = ngraph::pattern::wrap_type(); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -32,7 +34,7 @@ ConcatTransformation::ConcatTransformation(const Params& params) : LayerTransfor return transform(*context, m); }; - auto m = std::make_shared(matcher, "ConcatTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/convert.cpp b/src/common/low_precision_transformations/src/convert.cpp index c9438dfa810..e56d3eaabd4 100644 --- a/src/common/low_precision_transformations/src/convert.cpp +++ b/src/common/low_precision_transformations/src/convert.cpp @@ -14,12 +14,14 @@ #include #include "low_precision/common/ie_lpt_exception.hpp" #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { ConvertTransformation::ConvertTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(ConvertTransformation); auto matcher = pattern::wrap_type(); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -30,7 +32,7 @@ ConvertTransformation::ConvertTransformation(const Params& params) : LayerTransf return transform(*context, m); }; - auto m = std::make_shared(matcher, "ConvertTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/convert_subtract_constant.cpp b/src/common/low_precision_transformations/src/convert_subtract_constant.cpp index 9a8fccbeb1c..0bfa25c22be 100644 --- a/src/common/low_precision_transformations/src/convert_subtract_constant.cpp +++ b/src/common/low_precision_transformations/src/convert_subtract_constant.cpp @@ -11,6 +11,7 @@ #include #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" using namespace ngraph; @@ -35,6 +36,7 @@ using namespace ngraph; // Multiply // ngraph::pass::low_precision::ConvertSubtractConstant::ConvertSubtractConstant(const std::vector& constantPrecisions) { + MATCHER_SCOPE(ConvertSubtractConstant); auto weightsConstantWrapper = ngraph::pattern::wrap_type(pattern::consumers_count(1)); auto weightsConvertWrapper = ngraph::pattern::wrap_type({ weightsConstantWrapper }, pattern::consumers_count(1)); auto subtractConstantWrapper = ngraph::pattern::wrap_type(pattern::consumers_count(1)); @@ -91,6 +93,6 @@ ngraph::pass::low_precision::ConvertSubtractConstant::ConvertSubtractConstant(co return true; }; - auto m = std::make_shared(multiplyWrapper, "ConvertSubtractConstant"); + auto m = std::make_shared(multiplyWrapper, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/convolution.cpp b/src/common/low_precision_transformations/src/convolution.cpp index 7079256b39b..e34fd320744 100644 --- a/src/common/low_precision_transformations/src/convolution.cpp +++ b/src/common/low_precision_transformations/src/convolution.cpp @@ -14,12 +14,14 @@ #include #include "low_precision/network_helper.hpp" #include +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { ConvolutionTransformation::ConvolutionTransformation(const Params& params) : WeightableLayerTransformation(params) { + MATCHER_SCOPE(ConvolutionTransformation); auto matcher = ngraph::pattern::wrap_type({ ngraph::pattern::wrap_type(), std::make_shared(OutputVector { @@ -37,7 +39,7 @@ ConvolutionTransformation::ConvolutionTransformation(const Params& params) : Wei return transform(*context, m); }; - auto m = std::make_shared(matcher, "ConvolutionTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/convolution_backprop_data.cpp b/src/common/low_precision_transformations/src/convolution_backprop_data.cpp index 54785670e51..2691e9fdab2 100644 --- a/src/common/low_precision_transformations/src/convolution_backprop_data.cpp +++ b/src/common/low_precision_transformations/src/convolution_backprop_data.cpp @@ -14,12 +14,14 @@ #include #include "low_precision/network_helper.hpp" #include +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { ConvolutionBackpropDataTransformation::ConvolutionBackpropDataTransformation(const Params& params) : WeightableLayerTransformation(params) { + MATCHER_SCOPE(ConvolutionBackpropDataTransformation); auto matcher = std::make_shared(OutputVector{ pattern::wrap_type({ pattern::wrap_type(), @@ -49,7 +51,7 @@ ConvolutionBackpropDataTransformation::ConvolutionBackpropDataTransformation(con return transform(*context, m); }; - auto m = std::make_shared(matcher, "ConvolutionBackpropDataTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/depth_to_space.cpp b/src/common/low_precision_transformations/src/depth_to_space.cpp index d186e42ddce..200ac68be05 100644 --- a/src/common/low_precision_transformations/src/depth_to_space.cpp +++ b/src/common/low_precision_transformations/src/depth_to_space.cpp @@ -7,12 +7,14 @@ #include #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" using namespace ngraph; using namespace ngraph::pass; using namespace ngraph::pass::low_precision; DepthToSpaceTransformation::DepthToSpaceTransformation(const Params& params) : TransparentBaseTransformation(params) { + MATCHER_SCOPE(DepthToSpaceTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -23,7 +25,7 @@ DepthToSpaceTransformation::DepthToSpaceTransformation(const Params& params) : T return transform(*context, m); }; - auto m = std::make_shared(matcher, "DepthToSpaceTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/fake_quantize.cpp b/src/common/low_precision_transformations/src/fake_quantize.cpp index eba74a217b8..b45336cc724 100644 --- a/src/common/low_precision_transformations/src/fake_quantize.cpp +++ b/src/common/low_precision_transformations/src/fake_quantize.cpp @@ -10,12 +10,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { FakeQuantizeTransformation::FakeQuantizeTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(FakeQuantizeTransformation); auto matcher = pattern::wrap_type(); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -27,7 +29,7 @@ FakeQuantizeTransformation::FakeQuantizeTransformation(const Params& params) : L return transform(*context, m); }; - auto m = std::make_shared(matcher, "FakeQuantizeTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/fake_quantize_decomposition.cpp b/src/common/low_precision_transformations/src/fake_quantize_decomposition.cpp index e6f34c00403..3cb612e588a 100644 --- a/src/common/low_precision_transformations/src/fake_quantize_decomposition.cpp +++ b/src/common/low_precision_transformations/src/fake_quantize_decomposition.cpp @@ -14,12 +14,14 @@ #include "low_precision/rt_info/intervals_alignment_attribute.hpp" #include "low_precision/rt_info/quantization_alignment_attribute.hpp" #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { FakeQuantizeDecompositionTransformation::FakeQuantizeDecompositionTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(FakeQuantizeDecompositionTransformation); auto matcher = pattern::wrap_type(); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -31,7 +33,7 @@ FakeQuantizeDecompositionTransformation::FakeQuantizeDecompositionTransformation return transform(*context, m); }; - auto m = std::make_shared(matcher, "FakeQuantizeDecompositionTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/fold_convert.cpp b/src/common/low_precision_transformations/src/fold_convert.cpp index 5efc648304f..9921037ec8b 100644 --- a/src/common/low_precision_transformations/src/fold_convert.cpp +++ b/src/common/low_precision_transformations/src/fold_convert.cpp @@ -8,14 +8,16 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { FoldConvertTransformation::FoldConvertTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(FoldConvertTransformation); auto subtract = pattern::wrap_type(); - auto matcher = std::make_shared(subtract, "FoldConvertTransformation"); + auto matcher = std::make_shared(subtract, matcher_name); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { auto op = m.get_match_root(); diff --git a/src/common/low_precision_transformations/src/fold_fake_quantize.cpp b/src/common/low_precision_transformations/src/fold_fake_quantize.cpp index 6fb8f1a6366..18cf19f7391 100644 --- a/src/common/low_precision_transformations/src/fold_fake_quantize.cpp +++ b/src/common/low_precision_transformations/src/fold_fake_quantize.cpp @@ -10,12 +10,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { FoldFakeQuantizeTransformation::FoldFakeQuantizeTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(FoldFakeQuantizeTransformation); auto fakeQuantize = pattern::wrap_type(); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -26,7 +28,7 @@ FoldFakeQuantizeTransformation::FoldFakeQuantizeTransformation(const Params& par return transform(*context, m); }; - auto m = std::make_shared(fakeQuantize, "FoldFakeQuantizeTransformation"); + auto m = std::make_shared(fakeQuantize, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/fuse_convert.cpp b/src/common/low_precision_transformations/src/fuse_convert.cpp index 98c75d9475a..875c56efea1 100644 --- a/src/common/low_precision_transformations/src/fuse_convert.cpp +++ b/src/common/low_precision_transformations/src/fuse_convert.cpp @@ -12,12 +12,14 @@ #include "low_precision/common/ie_lpt_exception.hpp" #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { FuseConvertTransformation::FuseConvertTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(FuseConvertTransformation); auto multiply = pattern::wrap_type({ pattern::wrap_type(), pattern::wrap_type() }); auto subtract = pattern::wrap_type({ pattern::wrap_type(), pattern::wrap_type() }); auto add = pattern::wrap_type({ pattern::wrap_type(), pattern::wrap_type() }); @@ -29,7 +31,7 @@ FuseConvertTransformation::FuseConvertTransformation(const Params& params) : Lay pattern::any_input()}); auto matcher = std::make_shared( std::make_shared(OutputVector{ multiply, subtract, add, fakeQuantize }), - "FuseConvertTransformation"); + matcher_name); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { auto op = m.get_match_root(); diff --git a/src/common/low_precision_transformations/src/fuse_multiply_to_fake_quantize.cpp b/src/common/low_precision_transformations/src/fuse_multiply_to_fake_quantize.cpp index 6bb211acf92..f2e17fb5bfd 100644 --- a/src/common/low_precision_transformations/src/fuse_multiply_to_fake_quantize.cpp +++ b/src/common/low_precision_transformations/src/fuse_multiply_to_fake_quantize.cpp @@ -9,12 +9,14 @@ #include "low_precision/rt_info/intervals_alignment_attribute.hpp" #include "low_precision/fake_quantize.hpp" #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { FuseMultiplyToFakeQuantizeTransformation::FuseMultiplyToFakeQuantizeTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(FuseMultiplyToFakeQuantizeTransformation); auto matcher = pattern::wrap_type(); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -25,7 +27,7 @@ FuseMultiplyToFakeQuantizeTransformation::FuseMultiplyToFakeQuantizeTransformati return transform(*context, m); }; - auto m = std::make_shared(matcher, "FuseMultiplyToFakeQuantizeTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/fuse_subtract_to_fake_quantize.cpp b/src/common/low_precision_transformations/src/fuse_subtract_to_fake_quantize.cpp index 16b94ae9f31..1a727681052 100644 --- a/src/common/low_precision_transformations/src/fuse_subtract_to_fake_quantize.cpp +++ b/src/common/low_precision_transformations/src/fuse_subtract_to_fake_quantize.cpp @@ -8,12 +8,14 @@ #include #include "low_precision/fake_quantize.hpp" #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { FuseSubtractToFakeQuantizeTransformation::FuseSubtractToFakeQuantizeTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(FuseSubtractToFakeQuantizeTransformation); auto matcher = pattern::wrap_type(); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -24,7 +26,7 @@ FuseSubtractToFakeQuantizeTransformation::FuseSubtractToFakeQuantizeTransformati return transform(*context, m); }; - auto m = std::make_shared(matcher, "FuseSubtractToFakeQuantizeTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/group_convolution.cpp b/src/common/low_precision_transformations/src/group_convolution.cpp index e631a89d7ca..5aed9f8766e 100644 --- a/src/common/low_precision_transformations/src/group_convolution.cpp +++ b/src/common/low_precision_transformations/src/group_convolution.cpp @@ -10,12 +10,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { GroupConvolutionTransformation::GroupConvolutionTransformation(const Params& params) : ConvolutionTransformation(params) { + MATCHER_SCOPE(GroupConvolutionTransformation); auto matcher = pattern::wrap_type(); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -26,7 +28,7 @@ GroupConvolutionTransformation::GroupConvolutionTransformation(const Params& par return transform(*context, m); }; - auto m = std::make_shared(matcher, "GroupConvolutionTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/interpolate.cpp b/src/common/low_precision_transformations/src/interpolate.cpp index 21fe0443005..23908dc6102 100644 --- a/src/common/low_precision_transformations/src/interpolate.cpp +++ b/src/common/low_precision_transformations/src/interpolate.cpp @@ -12,12 +12,14 @@ #include #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" using namespace ngraph; using namespace ngraph::pass; using namespace ngraph::pass::low_precision; InterpolateTransformation::InterpolateTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(InterpolateTransformation); auto mul = pattern::wrap_type(); auto interpolate1 = pattern::wrap_type({ @@ -45,7 +47,7 @@ InterpolateTransformation::InterpolateTransformation(const Params& params) : Lay auto matcher = std::make_shared( std::make_shared(OutputVector{ interpolate1, interpolate4, interpolate4_2 }), - "InterpolateTransformation"); + matcher_name); this->register_matcher(matcher, callback); } diff --git a/src/common/low_precision_transformations/src/layer_transformation.cpp b/src/common/low_precision_transformations/src/layer_transformation.cpp index 54b85a93e0a..b469937f492 100644 --- a/src/common/low_precision_transformations/src/layer_transformation.cpp +++ b/src/common/low_precision_transformations/src/layer_transformation.cpp @@ -16,6 +16,7 @@ #include #include #include +#include "itt.hpp" namespace ngraph { namespace pass { @@ -433,6 +434,7 @@ void LayerTransformation::updateOutput( } void LayerTransformation::addPattern(ngraph::pass::GraphRewrite& pass, TransformationContext& context, std::shared_ptr patternRoot) { + MATCHER_SCOPE(SingleNodeMatcher); ngraph::graph_rewrite_callback internal_callback = [this, &context](ngraph::pattern::Matcher &m) { const bool result = transform(context, m); (void)result; @@ -449,7 +451,7 @@ void LayerTransformation::addPattern(ngraph::pass::GraphRewrite& pass, Transform return false; }; // TODO: better name for matcher? required? - auto m = std::make_shared(patternRoot, "SingleNodeMatcher"); + auto m = std::make_shared(patternRoot, matcher_name); NGRAPH_SUPPRESS_DEPRECATED_START pass.add_matcher(m, internal_callback, ngraph::pass::PassProperty::CHANGE_DYNAMIC_STATE); NGRAPH_SUPPRESS_DEPRECATED_END diff --git a/src/common/low_precision_transformations/src/low_precision.cpp b/src/common/low_precision_transformations/src/low_precision.cpp index 4747593d683..ef20fa0b1c7 100644 --- a/src/common/low_precision_transformations/src/low_precision.cpp +++ b/src/common/low_precision_transformations/src/low_precision.cpp @@ -78,6 +78,8 @@ #include "low_precision/fuse_multiply_to_fake_quantize.hpp" #include "low_precision/multiply_to_group_convolution.hpp" +#include "itt.hpp" + ngraph::pass::low_precision::LowPrecision::LowPrecision( const std::vector& precisionRestrictions, const std::vector& quantizationRestrictions, @@ -91,6 +93,7 @@ using namespace ngraph::pass::low_precision; template void make_matcher_type_relaxed(ngraph::pass::GraphRewrite* transformation) { + MATCHER_SCOPE(TypeRelaxedReplacer); using namespace ngraph; auto is_op_type = [](std::shared_ptr n) { @@ -127,7 +130,7 @@ void make_matcher_type_relaxed(ngraph::pass::GraphRewrite* transformation) { return true; }; - auto m = std::make_shared(p_node, "TypeRelaxedReplacer"); + auto m = std::make_shared(p_node, matcher_name); NGRAPH_SUPPRESS_DEPRECATED_START transformation->add_matcher(m, callback, ngraph::pass::PassProperty::CHANGE_DYNAMIC_STATE); NGRAPH_SUPPRESS_DEPRECATED_END @@ -163,6 +166,7 @@ MarkupOptimizations::MarkupOptimizations( params(params) {} bool ngraph::pass::low_precision::MarkupOptimizations::run_on_model(const std::shared_ptr& f) { + RUN_ON_FUNCTION_SCOPE(MarkupOptimizations); ngraph::pass::Manager markup(get_pass_config()); markup.set_per_pass_validation(false); markup.register_pass(params.defaultPrecisions); @@ -185,6 +189,7 @@ bool ngraph::pass::low_precision::MarkupOptimizations::run_on_model(const std::s } bool ngraph::pass::low_precision::LowPrecision::run_on_model(const std::shared_ptr& f) { + RUN_ON_FUNCTION_SCOPE(LowPrecision); OV_ITT_SCOPE(FIRST_INFERENCE, itt::domains::LPT_LT, "LowPrecision"); auto passConfig = get_pass_config(); diff --git a/src/common/low_precision_transformations/src/markup_avg_pool_precision_preserved.cpp b/src/common/low_precision_transformations/src/markup_avg_pool_precision_preserved.cpp index 4abed33cc39..f6e105f2122 100644 --- a/src/common/low_precision_transformations/src/markup_avg_pool_precision_preserved.cpp +++ b/src/common/low_precision_transformations/src/markup_avg_pool_precision_preserved.cpp @@ -9,6 +9,7 @@ #include "low_precision/rt_info/avg_pool_precision_preserved_attribute.hpp" #include "low_precision/propagate_through_precision_preserved.hpp" #include "low_precision/update_shared_precision_preserved.hpp" +#include "itt.hpp" using namespace ngraph; @@ -16,6 +17,7 @@ ngraph::pass::low_precision::MarkupAvgPoolPrecisionPreserved::MarkupAvgPoolPreci : defaultPrecisions(defaultPrecisions) {} bool ngraph::pass::low_precision::MarkupAvgPoolPrecisionPreserved::run_on_model(const std::shared_ptr& f) { + RUN_ON_FUNCTION_SCOPE(MarkupAvgPoolPrecisionPreserved); ngraph::pass::Manager manager; manager.set_per_pass_validation(false); std::shared_ptr markupAvgPoolPrecision = manager.register_pass(); diff --git a/src/common/low_precision_transformations/src/markup_can_be_quantized.cpp b/src/common/low_precision_transformations/src/markup_can_be_quantized.cpp index c4dd41e9e8b..42846ede90f 100644 --- a/src/common/low_precision_transformations/src/markup_can_be_quantized.cpp +++ b/src/common/low_precision_transformations/src/markup_can_be_quantized.cpp @@ -13,6 +13,7 @@ #include "low_precision/group_convolution.hpp" #include "low_precision/network_helper.hpp" #include "low_precision/rt_info/precisions_attribute.hpp" +#include "itt.hpp" using namespace ngraph; @@ -20,6 +21,7 @@ ngraph::pass::low_precision::MarkupCanBeQuantized::MarkupCanBeQuantized(const st : defaultPrecisions(defaultPrecisions) {} bool ngraph::pass::low_precision::MarkupCanBeQuantized::run_on_model(const std::shared_ptr& f) { + RUN_ON_FUNCTION_SCOPE(MarkupCanBeQuantized); auto setEmptyPrecisions = [](const std::shared_ptr& node) { for (auto& input : node->inputs()) { auto& rt = input.get_rt_info(); diff --git a/src/common/low_precision_transformations/src/markup_per_tensor_quantization.cpp b/src/common/low_precision_transformations/src/markup_per_tensor_quantization.cpp index 2aca084eddf..88d1fada809 100644 --- a/src/common/low_precision_transformations/src/markup_per_tensor_quantization.cpp +++ b/src/common/low_precision_transformations/src/markup_per_tensor_quantization.cpp @@ -9,6 +9,7 @@ #include #include #include "low_precision/rt_info/per_tensor_quantization_attribute.hpp" +#include "itt.hpp" using namespace ngraph; @@ -29,6 +30,7 @@ ngraph::pass::low_precision::MarkupPerTensorQuantization::MarkupPerTensorQuantiz } bool ngraph::pass::low_precision::MarkupPerTensorQuantization::run_on_model(const std::shared_ptr& f) { + RUN_ON_FUNCTION_SCOPE(MarkupPerTensorQuantization); auto setRestriction = [](const std::shared_ptr& node, const std::vector& restrictedPorts) { auto createAttribute = [](Input& input){ auto &rt = input.get_rt_info(); diff --git a/src/common/low_precision_transformations/src/markup_precisions.cpp b/src/common/low_precision_transformations/src/markup_precisions.cpp index 9643864290f..b820e08edde 100644 --- a/src/common/low_precision_transformations/src/markup_precisions.cpp +++ b/src/common/low_precision_transformations/src/markup_precisions.cpp @@ -16,6 +16,7 @@ #include "low_precision/network_helper.hpp" #include "low_precision/rt_info/precisions_attribute.hpp" #include "low_precision/rt_info/precision_preserved_attribute.hpp" +#include "itt.hpp" using namespace ngraph; @@ -64,6 +65,7 @@ void setRestriction( } // namespace bool ngraph::pass::low_precision::MarkupPrecisions::run_on_model(const std::shared_ptr& f) { + RUN_ON_FUNCTION_SCOPE(MarkupPrecisions); for (const std::shared_ptr& node : f->get_ordered_ops()) { if (node->get_input_size() == 0) { continue; diff --git a/src/common/low_precision_transformations/src/mat_mul.cpp b/src/common/low_precision_transformations/src/mat_mul.cpp index dba673c6512..667fe2f1178 100644 --- a/src/common/low_precision_transformations/src/mat_mul.cpp +++ b/src/common/low_precision_transformations/src/mat_mul.cpp @@ -13,12 +13,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" using namespace ngraph; using namespace ngraph::pass; using namespace ngraph::pass::low_precision; MatMulTransformation::MatMulTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(MatMulTransformation); auto mul1 = pattern::wrap_type(); auto mul2 = pattern::wrap_type(); auto fq2 = pattern::wrap_type(); @@ -32,7 +34,7 @@ MatMulTransformation::MatMulTransformation(const Params& params) : LayerTransfor return transform(*context, m); }; - auto m = std::make_shared(matcher, "MatMulTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/max_pool.cpp b/src/common/low_precision_transformations/src/max_pool.cpp index 3ba9d842097..0cecf09556c 100644 --- a/src/common/low_precision_transformations/src/max_pool.cpp +++ b/src/common/low_precision_transformations/src/max_pool.cpp @@ -11,12 +11,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { MaxPoolTransformation::MaxPoolTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(MaxPoolTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -27,7 +29,7 @@ MaxPoolTransformation::MaxPoolTransformation(const Params& params) : LayerTransf return transform(*context, m); }; - auto m = std::make_shared(matcher, "MaxPoolTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/move_fake_quantize.cpp b/src/common/low_precision_transformations/src/move_fake_quantize.cpp index b6fd023ce65..a1ccbb43dd6 100644 --- a/src/common/low_precision_transformations/src/move_fake_quantize.cpp +++ b/src/common/low_precision_transformations/src/move_fake_quantize.cpp @@ -14,12 +14,14 @@ #include "low_precision/concat.hpp" #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { MoveFakeQuantize::MoveFakeQuantize(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(MaxPoolTransformation); const auto concat = ngraph::pattern::wrap_type(pattern::consumers_count(1)); const auto operation = ngraph::pattern::wrap_type({ concat }); const auto input_low = ngraph::pattern::wrap_type(); @@ -48,7 +50,7 @@ MoveFakeQuantize::MoveFakeQuantize(const Params& params) : LayerTransformation(p auto m = std::make_shared( std::make_shared(OutputVector{fq, fq_with_operation}), - "MoveFakeQuantize"); + matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/multiply.cpp b/src/common/low_precision_transformations/src/multiply.cpp index 9b95281c110..0e827102ab5 100644 --- a/src/common/low_precision_transformations/src/multiply.cpp +++ b/src/common/low_precision_transformations/src/multiply.cpp @@ -16,12 +16,14 @@ #include "low_precision/common/ie_lpt_exception.hpp" #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { MultiplyTransformation::MultiplyTransformation(const Params& params) : EltwiseBaseTransformation(params) { + MATCHER_SCOPE(MultiplyTransformation); auto matcher = pattern::wrap_type(); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -32,7 +34,7 @@ MultiplyTransformation::MultiplyTransformation(const Params& params) : EltwiseBa return transform(*context, m); }; - auto m = std::make_shared(matcher, "MultiplyTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/multiply_to_group_convolution.cpp b/src/common/low_precision_transformations/src/multiply_to_group_convolution.cpp index 7407c68c53a..ca98a0934ca 100644 --- a/src/common/low_precision_transformations/src/multiply_to_group_convolution.cpp +++ b/src/common/low_precision_transformations/src/multiply_to_group_convolution.cpp @@ -7,6 +7,7 @@ #include #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { @@ -15,6 +16,7 @@ namespace low_precision { MultiplyToGroupConvolutionTransformation::MultiplyToGroupConvolutionTransformation( const Params& params, const OperationPrecisionRestriction::PrecisionsByPort& restrictions) : LayerTransformation(params), restrictions(restrictions), groupSize(1ul) { + MATCHER_SCOPE(MultiplyToGroupConvolutionTransformation); auto matcher = pattern::wrap_type(); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -25,7 +27,7 @@ MultiplyToGroupConvolutionTransformation::MultiplyToGroupConvolutionTransformati return transform(*context, m); }; - auto m = std::make_shared(matcher, "MultiplyToGroupConvolutionTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/mvn.cpp b/src/common/low_precision_transformations/src/mvn.cpp index 6640f78b125..85ac98df148 100644 --- a/src/common/low_precision_transformations/src/mvn.cpp +++ b/src/common/low_precision_transformations/src/mvn.cpp @@ -18,6 +18,7 @@ #include "low_precision/network_helper.hpp" #include "ngraph/opsets/opset6.hpp" +#include "itt.hpp" using namespace ngraph; using namespace ngraph::pass; @@ -41,6 +42,7 @@ std::shared_ptr createNewScalesConst(const ngraph::op::Con } // namespace mvn MVNTransformation::MVNTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(MVNTransformation); auto matcher = std::make_shared(OutputVector{ pattern::wrap_type({ pattern::wrap_type() }), pattern::wrap_type({ pattern::wrap_type(), pattern::wrap_type() }) @@ -54,7 +56,7 @@ MVNTransformation::MVNTransformation(const Params& params) : LayerTransformation return transform(*context, m); }; - auto m = std::make_shared(matcher, "MVNTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/normalize_l2.cpp b/src/common/low_precision_transformations/src/normalize_l2.cpp index 11df9947c0e..f9290eb11be 100644 --- a/src/common/low_precision_transformations/src/normalize_l2.cpp +++ b/src/common/low_precision_transformations/src/normalize_l2.cpp @@ -14,6 +14,7 @@ #include "ngraph/type/element_type.hpp" #include "ngraph/type/element_type_traits.hpp" #include "low_precision/network_helper.hpp" +#include "itt.hpp" using namespace ngraph; using namespace ngraph::pass; @@ -37,6 +38,7 @@ std::shared_ptr createNewScalesConst(const ngraph::op::Con } // namespace normalize_l2 NormalizeL2Transformation::NormalizeL2Transformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(NormalizeL2Transformation); auto matcher = pattern::wrap_type({ pattern::wrap_type(), pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -47,7 +49,7 @@ NormalizeL2Transformation::NormalizeL2Transformation(const Params& params) : Lay return transform(*context, m); }; - auto m = std::make_shared(matcher, "NormalizeL2Transformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/pad.cpp b/src/common/low_precision_transformations/src/pad.cpp index 88d43ad84a9..ef0491ee905 100644 --- a/src/common/low_precision_transformations/src/pad.cpp +++ b/src/common/low_precision_transformations/src/pad.cpp @@ -9,12 +9,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { PadTransformation::PadTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(PadTransformation); auto mul = pattern::wrap_type(); auto padsBegin = pattern::wrap_type(); auto padsEnd = pattern::wrap_type(); @@ -29,7 +31,7 @@ PadTransformation::PadTransformation(const Params& params) : LayerTransformation return transform(*context, m); }; - auto m = std::make_shared(matcher, "PadTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/prelu.cpp b/src/common/low_precision_transformations/src/prelu.cpp index bd345df3588..fa0018cc5e8 100644 --- a/src/common/low_precision_transformations/src/prelu.cpp +++ b/src/common/low_precision_transformations/src/prelu.cpp @@ -12,12 +12,14 @@ #include "low_precision/common/ie_lpt_exception.hpp" #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { PReluTransformation::PReluTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(PReluTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type(), pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -28,7 +30,7 @@ PReluTransformation::PReluTransformation(const Params& params) : LayerTransforma return transform(*context, m); }; - auto m = std::make_shared(matcher, "PReluTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/propagate_precisions.cpp b/src/common/low_precision_transformations/src/propagate_precisions.cpp index 63c03c67b18..5c23face13f 100644 --- a/src/common/low_precision_transformations/src/propagate_precisions.cpp +++ b/src/common/low_precision_transformations/src/propagate_precisions.cpp @@ -11,6 +11,7 @@ #include "low_precision/rt_info/precisions_attribute.hpp" #include "low_precision/propagate_through_precision_preserved.hpp" #include "low_precision/propagate_to_input.hpp" +#include "itt.hpp" using namespace ngraph; using namespace ngraph::pass::low_precision; @@ -18,6 +19,7 @@ using namespace ngraph::pass::low_precision; ngraph::pass::low_precision::PropagatePrecisions::PropagatePrecisions(const AttributeParameters& params) : params(params) {} bool ngraph::pass::low_precision::PropagatePrecisions::run_on_model(const std::shared_ptr& f) { + RUN_ON_FUNCTION_SCOPE(PropagatePrecisions); ngraph::pass::Manager manager; manager.set_per_pass_validation(false); std::shared_ptr precisionsPropagation = manager.register_pass(); diff --git a/src/common/low_precision_transformations/src/pull_reshape_through_dequantization.cpp b/src/common/low_precision_transformations/src/pull_reshape_through_dequantization.cpp index 3dd17461f23..899fb1fe001 100644 --- a/src/common/low_precision_transformations/src/pull_reshape_through_dequantization.cpp +++ b/src/common/low_precision_transformations/src/pull_reshape_through_dequantization.cpp @@ -12,6 +12,7 @@ #include #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" using namespace ngraph; @@ -94,6 +95,7 @@ ngraph::pass::low_precision::PullReshapeThroughDequantization::PullReshapeThroug const auto weights = ngraph::pattern::wrap_type(pattern::type_matches_any(inputPrecisions)); const auto convert = ngraph::pattern::wrap_type({ weights }); + MATCHER_SCOPE(PullReshapeThroughDequantization); const auto subtractValues = std::make_shared(OutputVector{ ngraph::pattern::wrap_type(), ngraph::pattern::wrap_type({ngraph::pattern::wrap_type()}) @@ -134,6 +136,6 @@ ngraph::pass::low_precision::PullReshapeThroughDequantization::PullReshapeThroug return true; }; - auto m = std::make_shared(reshapeWrapper, "PullReshapeThroughDequantization"); + auto m = std::make_shared(reshapeWrapper, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/pull_transpose_through_dequantization.cpp b/src/common/low_precision_transformations/src/pull_transpose_through_dequantization.cpp index 9d8551d9cfe..b4cd49b0988 100644 --- a/src/common/low_precision_transformations/src/pull_transpose_through_dequantization.cpp +++ b/src/common/low_precision_transformations/src/pull_transpose_through_dequantization.cpp @@ -13,6 +13,7 @@ #include #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" using namespace ngraph; @@ -92,6 +93,7 @@ ngraph::pass::low_precision::PullTransposeThroughDequantization::PullTransposeTh const auto weights = ngraph::pattern::wrap_type(pattern::type_matches_any(inputPrecisions)); const auto convert = ngraph::pattern::wrap_type({ weights }); + MATCHER_SCOPE(PullTransposeThroughDequantization); const auto subtractValues = std::make_shared(OutputVector{ ngraph::pattern::wrap_type(), ngraph::pattern::wrap_type({ngraph::pattern::wrap_type()}) @@ -127,6 +129,6 @@ ngraph::pass::low_precision::PullTransposeThroughDequantization::PullTransposeTh return true; }; - auto m = std::make_shared(matcherTranspose, "PullTransposeThroughDequantization"); + auto m = std::make_shared(matcherTranspose, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/reduce_max.cpp b/src/common/low_precision_transformations/src/reduce_max.cpp index c748a0e4951..f0ade04a425 100644 --- a/src/common/low_precision_transformations/src/reduce_max.cpp +++ b/src/common/low_precision_transformations/src/reduce_max.cpp @@ -8,12 +8,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { ReduceMaxTransformation::ReduceMaxTransformation(const Params& params) : ReduceBaseTransformation(params) { + MATCHER_SCOPE(ReduceMaxTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type(), pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -24,7 +26,7 @@ ReduceMaxTransformation::ReduceMaxTransformation(const Params& params) : ReduceB return transform(*context, m); }; - auto m = std::make_shared(matcher, "ReduceMaxTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/reduce_mean.cpp b/src/common/low_precision_transformations/src/reduce_mean.cpp index 5dfc25dc325..8861f5a2987 100644 --- a/src/common/low_precision_transformations/src/reduce_mean.cpp +++ b/src/common/low_precision_transformations/src/reduce_mean.cpp @@ -8,12 +8,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { ReduceMeanTransformation::ReduceMeanTransformation(const Params& params) : ReduceBaseTransformation(params) { + MATCHER_SCOPE(ReduceMeanTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type(), pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -24,7 +26,7 @@ ReduceMeanTransformation::ReduceMeanTransformation(const Params& params) : Reduc return transform(*context, m); }; - auto m = std::make_shared(matcher, "ReduceMeanTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/reduce_min.cpp b/src/common/low_precision_transformations/src/reduce_min.cpp index 04524867b7b..6e9b712d4e1 100644 --- a/src/common/low_precision_transformations/src/reduce_min.cpp +++ b/src/common/low_precision_transformations/src/reduce_min.cpp @@ -8,12 +8,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { ReduceMinTransformation::ReduceMinTransformation(const Params& params) : ReduceBaseTransformation(params) { + MATCHER_SCOPE(ReduceMinTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type(), pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -24,7 +26,7 @@ ReduceMinTransformation::ReduceMinTransformation(const Params& params) : ReduceB return transform(*context, m); }; - auto m = std::make_shared(matcher, "ReduceMinTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/reduce_sum.cpp b/src/common/low_precision_transformations/src/reduce_sum.cpp index 0b5565eb1c4..8e37f945998 100644 --- a/src/common/low_precision_transformations/src/reduce_sum.cpp +++ b/src/common/low_precision_transformations/src/reduce_sum.cpp @@ -8,12 +8,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { ReduceSumTransformation::ReduceSumTransformation(const Params& params) : ReduceBaseTransformation(params) { + MATCHER_SCOPE(ReduceSumTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type(), pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -24,7 +26,7 @@ ReduceSumTransformation::ReduceSumTransformation(const Params& params) : ReduceB return transform(*context, m); }; - auto m = std::make_shared(matcher, "ReduceSumTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/relu.cpp b/src/common/low_precision_transformations/src/relu.cpp index fb49d8f1241..eb032f86c9b 100644 --- a/src/common/low_precision_transformations/src/relu.cpp +++ b/src/common/low_precision_transformations/src/relu.cpp @@ -12,12 +12,14 @@ #include "low_precision/common/ie_lpt_exception.hpp" #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { ReluTransformation::ReluTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(ReluTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -28,7 +30,7 @@ ReluTransformation::ReluTransformation(const Params& params) : LayerTransformati return transform(*context, m); }; - auto m = std::make_shared(matcher, "ReluTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/reshape.cpp b/src/common/low_precision_transformations/src/reshape.cpp index b2a3fc0aad1..d9dfbdf3f05 100644 --- a/src/common/low_precision_transformations/src/reshape.cpp +++ b/src/common/low_precision_transformations/src/reshape.cpp @@ -16,12 +16,14 @@ #include "low_precision/common/ie_lpt_exception.hpp" #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { ReshapeTransformation::ReshapeTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(ReshapeTransformation); auto input = pattern::any_input(); auto mul_const_m = pattern::wrap_type(); auto mul_m = pattern::wrap_type({ input, mul_const_m }); @@ -48,7 +50,7 @@ ReshapeTransformation::ReshapeTransformation(const Params& params) : LayerTransf return transform(*context, m); }; - auto m = std::make_shared(matcher, "ReshapeTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/shuffle_channels.cpp b/src/common/low_precision_transformations/src/shuffle_channels.cpp index 68c92e7821c..7383717c298 100644 --- a/src/common/low_precision_transformations/src/shuffle_channels.cpp +++ b/src/common/low_precision_transformations/src/shuffle_channels.cpp @@ -11,12 +11,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { ShuffleChannelsTransformation::ShuffleChannelsTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(ShuffleChannelsTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -27,7 +29,7 @@ ShuffleChannelsTransformation::ShuffleChannelsTransformation(const Params& param return transform(*context, m); }; - auto m = std::make_shared(matcher, "ShuffleChannelsTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/split.cpp b/src/common/low_precision_transformations/src/split.cpp index 16f92b8e014..8587cb96f0c 100644 --- a/src/common/low_precision_transformations/src/split.cpp +++ b/src/common/low_precision_transformations/src/split.cpp @@ -8,12 +8,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { SplitTransformation::SplitTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(SplitTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type(), pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -24,7 +26,7 @@ SplitTransformation::SplitTransformation(const Params& params) : LayerTransforma return transform(*context, m); }; - auto m = std::make_shared(matcher, "SplitTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/squeeze.cpp b/src/common/low_precision_transformations/src/squeeze.cpp index b32723f7253..742ff6f933d 100644 --- a/src/common/low_precision_transformations/src/squeeze.cpp +++ b/src/common/low_precision_transformations/src/squeeze.cpp @@ -11,12 +11,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { SqueezeTransformation::SqueezeTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(SqueezeTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type(), pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -27,7 +29,7 @@ SqueezeTransformation::SqueezeTransformation(const Params& params) : LayerTransf return transform(*context, m); }; - auto m = std::make_shared(matcher, "SqueezeTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/strided_slice.cpp b/src/common/low_precision_transformations/src/strided_slice.cpp index b33f7f6edf5..743218c3ba5 100644 --- a/src/common/low_precision_transformations/src/strided_slice.cpp +++ b/src/common/low_precision_transformations/src/strided_slice.cpp @@ -9,6 +9,7 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { @@ -80,6 +81,7 @@ std::shared_ptr stridedSliceDeqConstant( } // namespace StridedSliceTransformation::StridedSliceTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(StridedSliceTransformation); auto matcher = ngraph::pattern::wrap_type(); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -90,7 +92,7 @@ StridedSliceTransformation::StridedSliceTransformation(const Params& params) : L return transform(*context, m); }; - auto m = std::make_shared(matcher, "StridedSliceTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/subtract.cpp b/src/common/low_precision_transformations/src/subtract.cpp index 8f64f07b3f7..0cc5d9a6aeb 100644 --- a/src/common/low_precision_transformations/src/subtract.cpp +++ b/src/common/low_precision_transformations/src/subtract.cpp @@ -16,12 +16,14 @@ #include "low_precision/common/ie_lpt_exception.hpp" #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { SubtractTransformation::SubtractTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(SubtractTransformation); auto convert = pattern::wrap_type(); auto multiply = pattern::wrap_type(); auto subParent = std::make_shared(OutputVector{ convert, multiply }); @@ -35,7 +37,7 @@ SubtractTransformation::SubtractTransformation(const Params& params) : LayerTran return transform(*context, m); }; - auto m = std::make_shared(subtract, "SubtractTransformation"); + auto m = std::make_shared(subtract, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/transpose.cpp b/src/common/low_precision_transformations/src/transpose.cpp index 4dd6944b1e1..71a1609da00 100644 --- a/src/common/low_precision_transformations/src/transpose.cpp +++ b/src/common/low_precision_transformations/src/transpose.cpp @@ -11,12 +11,14 @@ #include "low_precision/common/ie_lpt_exception.hpp" #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { TransposeTransformation::TransposeTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(TransposeTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type(), pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -27,7 +29,7 @@ TransposeTransformation::TransposeTransformation(const Params& params) : LayerTr return transform(*context, m); }; - auto m = std::make_shared(matcher, "TransposeTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/unsqueeze.cpp b/src/common/low_precision_transformations/src/unsqueeze.cpp index 191386a5219..e84f1db0539 100644 --- a/src/common/low_precision_transformations/src/unsqueeze.cpp +++ b/src/common/low_precision_transformations/src/unsqueeze.cpp @@ -11,12 +11,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { UnsqueezeTransformation::UnsqueezeTransformation(const Params& params) : LayerTransformation(params) { + MATCHER_SCOPE(UnsqueezeTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type(), pattern::wrap_type() }); ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { @@ -27,7 +29,7 @@ UnsqueezeTransformation::UnsqueezeTransformation(const Params& params) : LayerTr return transform(*context, m); }; - auto m = std::make_shared(matcher, "UnsqueezeTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/common/low_precision_transformations/src/variadic_split.cpp b/src/common/low_precision_transformations/src/variadic_split.cpp index 26318bc9a11..fc0644edfbb 100644 --- a/src/common/low_precision_transformations/src/variadic_split.cpp +++ b/src/common/low_precision_transformations/src/variadic_split.cpp @@ -8,12 +8,14 @@ #include #include "low_precision/network_helper.hpp" +#include "itt.hpp" namespace ngraph { namespace pass { namespace low_precision { VariadicSplitTransformation::VariadicSplitTransformation(const Params& params) : SplitTransformation(params) { + MATCHER_SCOPE(VariadicSplitTransformation); auto matcher = pattern::wrap_type({ pattern::wrap_type(), pattern::wrap_type(), @@ -27,7 +29,7 @@ VariadicSplitTransformation::VariadicSplitTransformation(const Params& params) : return transform(*context, m); }; - auto m = std::make_shared(matcher, "VariadicSplitTransformation"); + auto m = std::make_shared(matcher, matcher_name); this->register_matcher(m, callback); } diff --git a/src/plugins/intel_cpu/src/itt.h b/src/plugins/intel_cpu/src/itt.h index cf80814bfcd..0d38d35d8c8 100644 --- a/src/plugins/intel_cpu/src/itt.h +++ b/src/plugins/intel_cpu/src/itt.h @@ -9,6 +9,9 @@ #pragma once +#include +#include + #include namespace ov { @@ -21,3 +24,13 @@ namespace domains { } } } + +#if defined(SELECTIVE_BUILD_ANALYZER) +# define CPU_LPT_SCOPE(region) OV_SCOPE(intel_cpu, region) +#elif defined(SELECTIVE_BUILD) +# define CPU_LPT_SCOPE(region) \ + if (OV_CC_SCOPE_IS_ENABLED(OV_PP_CAT3(intel_cpu, _, region)) == 0) \ + throw ngraph::ngraph_error(std::string(OV_PP_TOSTRING(OV_PP_CAT3(ngraph_op, _, region))) + " is disabled!") +#else +# define CPU_LPT_SCOPE(region) OV_ITT_SCOPED_TASK(ov::intel_cpu::itt::domains::intel_cpu, OV_PP_TOSTRING(region)) +#endif diff --git a/src/plugins/intel_cpu/src/plugin.cpp b/src/plugins/intel_cpu/src/plugin.cpp index d5ab9fafc64..87c3a7aa521 100644 --- a/src/plugins/intel_cpu/src/plugin.cpp +++ b/src/plugins/intel_cpu/src/plugin.cpp @@ -131,6 +131,7 @@ #endif #include +#include using namespace InferenceEngine; @@ -185,6 +186,7 @@ static void TransformationUpToCPUSpecificOpSet(std::shared_ptr auto defaultPrecisions = useLpt ? ngraph::pass::low_precision::precision_set::int8_support : std::vector{}; bool hasINT16orINT32Levels = false; if (useLpt) { + CPU_LPT_SCOPE(LowPrecisionTransformations_Part1); hasINT16orINT32Levels = ngraph::pass::low_precision::LowPrecision::isFQLevelsPresent( nGraphFunc, {ngraph::pass::low_precision::levels::int16, ngraph::pass::low_precision::levels::int16_narrow_range, @@ -236,6 +238,7 @@ static void TransformationUpToCPUSpecificOpSet(std::shared_ptr manager.register_pass(); if (useLpt) { + CPU_LPT_SCOPE(LowPrecisionTransformations_Part2); manager.register_pass(defaultPrecisions); } manager.register_pass(); @@ -412,6 +415,7 @@ static void TransformationUpToCPUSpecificOpSet(std::shared_ptr pass_config->enable(); if (useLpt) { + CPU_LPT_SCOPE(LowPrecisionTransformations_Part3); pass_config->set_callback([](const_node_ptr &node) -> bool { @@ -432,6 +436,7 @@ static void TransformationUpToCPUSpecificOpSet(std::shared_ptr using namespace ngraph::pass::low_precision; if (useLpt) { + CPU_LPT_SCOPE(LowPrecisionTransformations_Part4); OV_ITT_SCOPE(FIRST_INFERENCE, itt::domains::intel_cpu_LT, "LowPrecisionTransformations"); auto supportedPrecisions = std::vector({ From f2af1ef88a3fabbde5adec60a032f9338d69465c Mon Sep 17 00:00:00 2001 From: Sergey Shlyapnikov Date: Thu, 31 Mar 2022 13:13:58 +0300 Subject: [PATCH 24/39] [GPU] Update memory location to __local in GPU Detection Output (#11209) * [GPU] Update memory location to __local in GPU Detection Output * Replace hardcoded stack size with JIT constant --- .../detection_output_kernel_ref.cpp | 26 ++++++++-- .../cl_kernels/detection_output_gpu_ref.cl | 47 +++++++++++++++---- 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/detection_output/detection_output_kernel_ref.cpp b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/detection_output/detection_output_kernel_ref.cpp index 0e4476ca142..e2b57ade263 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/detection_output/detection_output_kernel_ref.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/detection_output/detection_output_kernel_ref.cpp @@ -212,6 +212,7 @@ KernelsData DetectionOutputKernelRef::GetKernelsData(const Params& params, const auto num_prior_boxes = (loc_feature_num / (num_loc_classes * prior_box_size)); auto max_wg = detectOutParams.engineInfo.maxWorkGroupSize; + constexpr size_t stack_size = 100; // The size of stack for QuickSort constexpr size_t buffer_bytes = 10; // The size of struct Scores in detection_output_gpu_ref.cl size_t buffer_stride = num_prior_boxes * buffer_bytes; size_t buffer_size = num_of_images * num_classes * buffer_stride; @@ -229,6 +230,7 @@ KernelsData DetectionOutputKernelRef::GetKernelsData(const Params& params, const auto cldnnJit = GetJitConstants(detectOutParams); auto entryPoint = GetEntryPoint(kernelName, detectOutParams.layerID, params, options, i); cldnnJit.AddConstant(MakeJitConstant("BUFFER_STRIDE", buffer_stride)); + cldnnJit.AddConstant(MakeJitConstant("QUICK_SORT_STACK_SIZE", stack_size)); if (i == 0) { if (detectOutParams.detectOutParams.decrease_label_id) { cldnnJit.AddConstant(MakeJitConstant("DO_STAGE_" + std::to_string(i) + "_MXNET", "true")); @@ -246,17 +248,25 @@ KernelsData DetectionOutputKernelRef::GetKernelsData(const Params& params, const MakeJitConstant("NUM_PRIOR_BLOCKS", num_score_block)}); } } else if (i == 1) { - if (detectOutParams.detectOutParams.decrease_label_id) { + if (detectOutParams.detectOutParams.decrease_label_id) { + // Always use local memory since LWS size is 1x1x16 (16 WI * 100 (stack size) * 4 (int size) = 6.25 KB of SLM memory) + cldnnJit.AddConstant(MakeJitConstant("USE_LOCAL_MEMORY_FOR_STACK", true)); cldnnJit.AddConstants({MakeJitConstant("DO_STAGE_" + std::to_string(i) + "_MXNET", "true"), MakeJitConstant("LOCAL_WORK_NUM", dispatchData.lws[2]), MakeJitConstant("PARTITION_STEP", GetPartitionStep(dispatchData.lws[2]))}); - } else { + } else { + // Limit local memory usage for two buffers: __range [LWS1 * LWS2 * 2 * 4 (int size) bytes] + // stack [LWS1 * LWS2 * 100 (stack_size) * 4 (int size) bytes] + auto req_local_mem_size = dispatchData.lws[1] * dispatchData.lws[2] * 2 * 4 + + dispatchData.lws[1] * dispatchData.lws[2] * stack_size * 4; + if (req_local_mem_size < detectOutParams.engineInfo.maxLocalMemSize) + cldnnJit.AddConstant(MakeJitConstant("USE_LOCAL_MEMORY_FOR_STACK", true)); cldnnJit.AddConstants({MakeJitConstant("DO_STAGE_" + std::to_string(i) + "_CAFFE", "true"), MakeJitConstant("LOCAL_CLASS_NUM", dispatchData.lws[1]), MakeJitConstant("LOCAL_WORK_NUM", dispatchData.lws[2]), MakeJitConstant("PARTITION_STEP", GetPartitionStep(dispatchData.lws[2]))}); - } - } else if (i == 2) { + } + } else if (i == 2) { if (detectOutParams.detectOutParams.decrease_label_id) { cldnnJit.AddConstant(MakeJitConstant("DO_STAGE_" + std::to_string(i) + "_MXNET", "true")); } else { @@ -295,10 +305,16 @@ KernelsData DetectionOutputKernelRef::GetKernelsData(const Params& params, const cldnnJit.AddConstant(MakeJitConstant("DO_STAGE_" + std::to_string(i) + "_CAFFE", "true")); } } - } else { + } else { if (detectOutParams.detectOutParams.decrease_label_id) { cldnnJit.AddConstant(MakeJitConstant("DO_STAGE_" + std::to_string(i) + "_MXNET", "true")); + // Always use local memory since LWS size is 1x1x1 + cldnnJit.AddConstant(MakeJitConstant("USE_LOCAL_MEMORY_FOR_STACK", true)); } else { + // Limit local memory usage for stack buffer [LWS0 * 100 (stack_size) * 4 (int size) bytes] + auto req_local_mem_size = dispatchData.lws[0] * stack_size * 4; + if (req_local_mem_size < detectOutParams.engineInfo.maxLocalMemSize) + cldnnJit.AddConstant(MakeJitConstant("USE_LOCAL_MEMORY_FOR_STACK", true)); cldnnJit.AddConstants({MakeJitConstant("DO_STAGE_" + std::to_string(i) + "_CAFFE", "true"), MakeJitConstant("LOCAL_BATCHES_NUM", dispatchData.lws[0])}); } diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/cl_kernels/detection_output_gpu_ref.cl b/src/plugins/intel_gpu/src/kernel_selector/core/cl_kernels/detection_output_gpu_ref.cl index 3eaf7008e21..dd93980e4d7 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/core/cl_kernels/detection_output_gpu_ref.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/core/cl_kernels/detection_output_gpu_ref.cl @@ -109,11 +109,16 @@ inline void FUNC(bubbleSortIterative)(__global SCORES_INFO* arr, int l, int h) { } } -inline void FUNC(quickSortIterative)(__global SCORES_INFO* arr, int l, int h, bool use_custom_comp) { +inline void FUNC(quickSortIterative)(__global SCORES_INFO* arr, + int l, int h, +#ifdef USE_LOCAL_MEMORY_FOR_STACK + __local int* stack, +#endif + bool use_custom_comp) { +#ifndef USE_LOCAL_MEMORY_FOR_STACK // Create an auxiliary stack - const int kStackSize = 100; - int stack[kStackSize]; - + int stack[QUICK_SORT_STACK_SIZE]; +#endif // initialize top of stack int top = -1; @@ -134,7 +139,7 @@ inline void FUNC(quickSortIterative)(__global SCORES_INFO* arr, int l, int h, bo // If there are elements on left side of pivot, // then push left side to stack if (p - 1 > l) { - if (top >= (kStackSize - 1)) { + if (top >= (QUICK_SORT_STACK_SIZE - 1)) { FUNC_CALL(bubbleSortIterative)(arr, l, p - 1); } else { stack[++top] = l; @@ -145,7 +150,7 @@ inline void FUNC(quickSortIterative)(__global SCORES_INFO* arr, int l, int h, bo // If there are elements on right side of pivot, // then push right side to stack if (p + 1 < h) { - if (top >= (kStackSize - 1)) { + if (top >= (QUICK_SORT_STACK_SIZE - 1)) { FUNC_CALL(bubbleSortIterative)(arr, p + 1, h); } else { stack[++top] = p + 1; @@ -415,6 +420,12 @@ KERNEL (detection_output_stage_1_sort_caffe)(__global uchar *buffer0, const int localClassId = get_local_id(1); __local int __range[LOCAL_CLASS_NUM][LOCAL_WORK_NUM * 2]; +#ifdef USE_LOCAL_MEMORY_FOR_STACK + // Create an auxiliary stack for QuickSort + __local int stack[QUICK_SORT_STACK_SIZE * LOCAL_CLASS_NUM * LOCAL_WORK_NUM]; + __local int *stack_pointer = stack + workItemId * QUICK_SORT_STACK_SIZE + localClassId * LOCAL_WORK_NUM * QUICK_SORT_STACK_SIZE; +#endif + const int scoresInfoNum = buffer1[batchId * NUM_CLASSES_ACC + classId]; __global SCORES_INFO *scoresList = (__global SCORES_INFO*)&buffer0[(batchId * NUM_CLASSES + classId) * BUFFER_STRIDE]; @@ -448,7 +459,11 @@ KERNEL (detection_output_stage_1_sort_caffe)(__global uchar *buffer0, const int begin_id = __range[localClassId][first_id]; const int end_id = __range[localClassId][first_id + 1]; if (begin_id < end_id) { +#ifdef USE_LOCAL_MEMORY_FOR_STACK + FUNC_CALL(quickSortIterative)(scoresList, begin_id, end_id, stack_pointer, true); +#else FUNC_CALL(quickSortIterative)(scoresList, begin_id, end_id, true); +#endif } barrier(CLK_GLOBAL_MEM_FENCE | CLK_LOCAL_MEM_FENCE); @@ -470,6 +485,9 @@ KERNEL (detection_output_stage_1_sort_mxnet)(__global uchar *buffer0, const int workItemId = get_global_id(2); __local int __range[LOCAL_WORK_NUM * 2]; + // Create an auxiliary stack for QuickSort + __local int stack[QUICK_SORT_STACK_SIZE]; + const int scoresInfoNum = buffer2[batchId * NUM_CLASSES_ACC + NUM_CLASSES]; if (scoresInfoNum < 2) return; @@ -507,7 +525,7 @@ KERNEL (detection_output_stage_1_sort_mxnet)(__global uchar *buffer0, const int begin_id = __range[first_id]; const int end_id = __range[first_id + 1]; if (begin_id < end_id) { - FUNC_CALL(quickSortIterative)(scoresList, begin_id, end_id, true); + FUNC_CALL(quickSortIterative)(scoresList, begin_id, end_id, stack, true); } barrier(CLK_GLOBAL_MEM_FENCE | CLK_LOCAL_MEM_FENCE); @@ -666,6 +684,12 @@ KERNEL (detection_output_stage_final_caffe)(__global INPUT0_TYPE* input_location __global int *buffer1) { const int batchId = get_global_id(0); +#ifdef USE_LOCAL_MEMORY_FOR_STACK + // Create an auxiliary stack for QuickSort + __local int stack[QUICK_SORT_STACK_SIZE]; + __local int *stack_pointer = stack + batchId * QUICK_SORT_STACK_SIZE; +#endif + const int total_det = FUNC_CALL(get_accumulated_detections)(buffer1, batchId); buffer1[batchId * NUM_CLASSES_ACC + NUM_CLASSES] = total_det; // the total number of detections is also stored in the extra space of buffer @@ -692,7 +716,11 @@ KERNEL (detection_output_stage_final_caffe)(__global INPUT0_TYPE* input_location buffer1[scores_size_offset + idx_class] = 0; } +#ifdef USE_LOCAL_MEMORY_FOR_STACK + FUNC_CALL(quickSortIterative)(scoresList + scores_offset, 0, num_det - 1, stack_pointer, true); +#else FUNC_CALL(quickSortIterative)(scoresList + scores_offset, 0, num_det - 1, true); +#endif // recalculate valid items for each class for (uint idx_num_det = 0; idx_num_det < KEEP_TOP_K; idx_num_det++) { @@ -788,6 +816,9 @@ KERNEL (detection_output_stage_final_mxnet)(__global INPUT0_TYPE* input_location __global uchar *buffer0, __global uchar *buffer1, __global int *buffer2) { + // Create an auxiliary stack for QuickSort + __local int stack[QUICK_SORT_STACK_SIZE]; + for (uint idx_image = 0; idx_image < NUM_OF_IMAGES; idx_image++) { __global SCORES_INFO *scoresList = (__global SCORES_INFO*)&buffer0[idx_image * BUFFER_STRIDE]; const int total_det = buffer2[idx_image * NUM_CLASSES_ACC + NUM_CLASSES]; @@ -805,7 +836,7 @@ KERNEL (detection_output_stage_final_mxnet)(__global INPUT0_TYPE* input_location num_det += acc_num; buffer2[scores_size_offset] = 0; } - FUNC_CALL(quickSortIterative)(scoresList, 0, num_det - 1, true); + FUNC_CALL(quickSortIterative)(scoresList, 0, num_det - 1, stack, true); for (uint idx_num_det = 0; idx_num_det < KEEP_TOP_K; idx_num_det++) { int scores_size_offset = idx_image * NUM_CLASSES_ACC + (int)scoresList[idx_num_det].classId; From 79e32722372ae28e1f98c482cf81960b351f2e75 Mon Sep 17 00:00:00 2001 From: Sergey Shlyapnikov Date: Thu, 31 Mar 2022 13:15:47 +0300 Subject: [PATCH 25/39] [GPU] Update eltwise calc_output_layout function and prevent output layouts invalidation after adding reorders for weights (#11073) --- src/plugins/intel_gpu/src/graph/eltwise.cpp | 16 ++++++++++++++-- .../src/graph/graph_optimizer/reorder_inputs.cpp | 4 +++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/eltwise.cpp b/src/plugins/intel_gpu/src/graph/eltwise.cpp index ec72f1e155e..3dc7422a1c0 100644 --- a/src/plugins/intel_gpu/src/graph/eltwise.cpp +++ b/src/plugins/intel_gpu/src/graph/eltwise.cpp @@ -18,13 +18,25 @@ primitive_type_id eltwise::type_id() { } layout eltwise_inst::calc_output_layout(eltwise_node const& node) { - auto input_node_layout = node.input().get_non_padded_output_layout(); + size_t primary_input_idx = 0; + if (node.input(primary_input_idx).is_constant()) { + for (size_t i = 1; i < node.get_dependencies().size(); i++) { + if (!node.input(i).is_constant()) { + primary_input_idx = i; + break; + } + } + } + auto input_node_layout = node.input(primary_input_idx).get_non_padded_output_layout(); auto output_type = node.get_primitive()->output_data_type ? *node.get_primitive()->output_data_type : input_node_layout.data_type; auto size = input_node_layout.size; auto format = input_node_layout.format; - for (size_t i = 1; i < node.inputs_count(); i++) { + for (size_t i = 0; i < node.inputs_count(); i++) { + if (i == primary_input_idx) + continue; + auto l = node.input(i).get_non_padded_output_layout(); size = tensor::max(size, l.size); if (l.format == format::b_fs_zyx_fsv16) // use optimized 5D diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_inputs.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_inputs.cpp index f9f50b56cf0..e30a76162f7 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_inputs.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_inputs.cpp @@ -559,6 +559,7 @@ void reorder_inputs::run(program& p, layout_optimizer& lo, reorder_factory& rf) layout{ weights_layout.data_type, preferred_format, weights_layout.size }); if (reorder.first) { p.add_intermediate(reorder.first, deconv_node, 1, !reorder.second); + p.get_or_create(reorder.first).recalc_output_layout(false); } } }; @@ -575,6 +576,7 @@ void reorder_inputs::run(program& p, layout_optimizer& lo, reorder_factory& rf) layout{ weights_layout.data_type, preferred_format, weights_layout.size }); if (reorder.first) { p.add_intermediate(reorder.first, conv_node, 1, !reorder.second); + p.get_or_create(reorder.first).recalc_output_layout(false); } } } @@ -708,7 +710,7 @@ void reorder_inputs::run(program& p, layout_optimizer& lo, reorder_factory& rf) reorder_input_and_weights_deconvolution, reorder_convolution, reorder_input_fully_connected); - } + } for (auto n : p.get_processing_order()) { if (n->is_in_data_flow() && fmt_map.count(n) != 0) { From 3578ee9c3f04bbf3c7d8c582c86065a34b846f5b Mon Sep 17 00:00:00 2001 From: Elizaveta Lobanova Date: Thu, 31 Mar 2022 13:21:27 +0300 Subject: [PATCH 26/39] [GNA] Remove extra FQ layers from the final network (#10599) * [GNA] Fuse all FakeQuantize layers with their previous layers * [GNA] Fuse FQ with previous layer if it's not required for precision change * [GNA] Fixed MatMulOverloadCorrectionTest --- src/plugins/intel_gna/backend/make_pwl.cpp | 4 +- .../intel_gna/frontend/scale_factor_calc.hpp | 3 +- .../intel_gna/optimizer/gna_pass_manager.cpp | 66 ++++++++----------- .../scale_factors_tests/eltwise_act_fq.cpp | 2 +- .../matmul_overload_correction.cpp | 6 +- .../weighable_layer_without_fq.cpp | 27 ++++---- .../src/subgraph/multiple_input_fq.cpp | 34 +++++++--- 7 files changed, 76 insertions(+), 66 deletions(-) diff --git a/src/plugins/intel_gna/backend/make_pwl.cpp b/src/plugins/intel_gna/backend/make_pwl.cpp index 6362cee472a..38c263082e0 100644 --- a/src/plugins/intel_gna/backend/make_pwl.cpp +++ b/src/plugins/intel_gna/backend/make_pwl.cpp @@ -229,7 +229,7 @@ void make_gna_pwl(const DnnActivation& fun, int32_t x_upper = INT32_MAX; int16_t y_lower = y_min; int16_t y_upper = y_max; - if (fun == kActFakeQuantize && fun.fqParams.set) { + if ((fun == kActFakeQuantize || fun == kActIdentity) && fun.fqParams.set) { x_lower = std::max(static_cast(*fun.fqParams.input_low * in_scale), static_cast(x_lower)); x_upper = std::min(static_cast(*fun.fqParams.input_high * in_scale), static_cast(x_upper)); y_lower = std::max(static_cast(*fun.fqParams.input_low * out_scale), static_cast(y_lower)); @@ -253,7 +253,7 @@ void make_gna_pwl(const DnnActivation& fun, x_upper = FLOAT_TO_INT32(y_upper * in_scale / out_scale); } } - } else if (fun == kActIdentity) { + } else if (fun == kActIdentity && !fun.fqParams.set) { if (x_lower < y_lower * in_scale / out_scale) x_lower = FLOAT_TO_INT32(y_lower * in_scale / out_scale); if (x_upper > y_upper * in_scale / out_scale) x_upper = FLOAT_TO_INT32(y_upper * in_scale / out_scale); if (y_lower < x_lower * out_scale / in_scale) y_lower = FLOAT_TO_INT16(x_lower * out_scale / in_scale); diff --git a/src/plugins/intel_gna/frontend/scale_factor_calc.hpp b/src/plugins/intel_gna/frontend/scale_factor_calc.hpp index 2f97459257f..1ef90832b8d 100644 --- a/src/plugins/intel_gna/frontend/scale_factor_calc.hpp +++ b/src/plugins/intel_gna/frontend/scale_factor_calc.hpp @@ -538,7 +538,8 @@ class ScaleFactorPerLayer { auto maxOutValue = quantizedParams->_dst_quant.GetMaxValues().front(); auto absMax = std::max(std::abs(minOutValue), std::abs(maxOutValue)); - result = CalculateScaleFactorFromStats(quantizedParams->_dst_quant.GetLevels(), minOutValue, maxOutValue); + auto levels = std::min(quantizedParams->_dst_quant.GetLevels(), static_cast(std::numeric_limits::max()) + 1); + result = CalculateScaleFactorFromStats(levels, minOutValue, maxOutValue); if (std::isinf(result) || fp32eq(absMax, 0.0f)) { result = max_activation_scale_factor; } diff --git a/src/plugins/intel_gna/optimizer/gna_pass_manager.cpp b/src/plugins/intel_gna/optimizer/gna_pass_manager.cpp index ef759b492b9..7ca2ba615b4 100644 --- a/src/plugins/intel_gna/optimizer/gna_pass_manager.cpp +++ b/src/plugins/intel_gna/optimizer/gna_pass_manager.cpp @@ -74,7 +74,8 @@ static const char softSignLayersCounter[] = "numSoftSignLayers"; static void insertDiagonalLayerBetween(InferenceEngine::CNNLayerPtr prevLayer, InferenceEngine::CNNLayerPtr nextLayer, std::shared_ptr passmanager, - float fillValue) { + float fillValue, + size_t in_data_idx = invalid_data_idx) { auto quantized = InferenceEngine::getInjectedData(prevLayer); auto diagName = std::string("SyntheticScaleShift_") + std::to_string(passmanager->getIntVar(diagonalLayersCounterName)++); gnalog() << "Inserted Diagonal Layer " << diagName <<" between: " << prevLayer->name << " and " << nextLayer->name << "\n" << std::flush; @@ -104,7 +105,7 @@ static void insertDiagonalLayerBetween(InferenceEngine::CNNLayerPtr prevLayer, getCreatorLayer(dataPtr) = diagonalWithQuant; diagonalWithQuant->outData.push_back(dataPtr); // actual insertion - CNNNetworkInsertLayer(prevLayer, nextLayer, diagonalWithQuant); + CNNNetworkInsertLayer(prevLayer, nextLayer, diagonalWithQuant, invalid_data_idx, in_data_idx); } /** @@ -137,7 +138,8 @@ static bool hasNextFuncLayer(const CNNLayerPtr layer) { }); } -static std::vector getCandidatesForIdentityInsertion(const CNNLayerPtr l, std::shared_ptr passmanager) { +static std::vector getCandidatesForIdentityInsertion(const CNNLayerPtr l, std::shared_ptr passmanager, + bool skipFq = false) { std::vector prevLayers; // skipping memory inputs and true inputs layers @@ -146,9 +148,9 @@ static std::vector getCandidatesForIdentityInsertion(const CNNLayer auto eltwise = dynamic_cast(l.get()); auto concat = dynamic_cast(l.get()); - auto PrevFunctionalLayer = [](CNNLayerPtr l, int idx = 0) { - auto prevLayer = CNNNetPrevLayerSkipCertain(l, idx, [](CNNLayerPtr ptr) { - return LayerInfo(ptr).isNonFunctional(); + auto PrevFunctionalLayer = [skipFq](CNNLayerPtr l, int idx = 0) { + auto prevLayer = CNNNetPrevLayerSkipCertain(l, idx, [skipFq](CNNLayerPtr ptr) { + return LayerInfo(ptr).isNonFunctional() || skipFq && LayerInfo(ptr).isFakeQuantize(); }); gnalog() << "CNNNetPrevLayerSkipCertain for :: " << l->name << "returned: " << prevLayer->name << std::endl; return prevLayer; @@ -308,7 +310,7 @@ void InsertDiagonalLayerPass::run() { continue; } auto prevDirectLayer = CNNNetPrevLayer(l, 0); - insertDiagonalLayerBetween(prevDirectLayer, l, getPassManager(), 1.f); + insertDiagonalLayerBetween(prevDirectLayer, l, getPassManager(), 1.f, 0); } } @@ -820,18 +822,6 @@ void InsertIdentityLayerPass::run() { gnalog() << "Inserted "<< identityLayer->name << " between: " << prev->name << " and " << true_layer->name << "\n" << std::flush; - // wether 1 identity or all outputs TODO possible grouping here, need to implement special grouped inserter - bool notAll = false; - for (auto && nextData : prev->outData) { - for (auto && nextLayer : getInputTo(nextData)) { - if (nextLayer.second.get() == l.get()) - continue; - if (getCandidatesForIdentityInsertion(nextLayer.second, getPassManager()).empty() && - hasNextFuncLayer(nextLayer.second)) { - notAll = true; - } - } - } // copy offset - to be used while connecting outputs if (prev->params.find("output_offset") != prev->params.end()) { identityLayer->params["output_offset"] = prev->params["output_offset"]; @@ -841,7 +831,7 @@ void InsertIdentityLayerPass::run() { identityLayer->params["original_num_rows"] = prev->params["original_num_rows"]; } - CNNNetworkInsertLayer(prev, notAll ? true_layer : CNNLayerPtr(nullptr), identityLayer); + CNNNetworkInsertLayer(prev, CNNLayerPtr(nullptr), identityLayer); } } } @@ -1341,6 +1331,12 @@ void InsertSplitAligningFilterPass::run() { if (getInputTo(splitOutput).empty()) { gnalog() << "Output port: " << splitOutIndex << " of " << l->name << " unconnected, skipping\n"; } else { + auto lastDimSize = GetDataDimSize(splitOutput, 1); + if (lastDimSize != outputSize) { + THROW_GNA_EXCEPTION << l->name << " Convolution Filter doesn't support these input dimensions: lastDimSize=" + << lastDimSize << ", outputSize=" << outputSize; + } + // this split output not beginning from 64 bytes aligned boundary - need to correct by aligning filter layer // insert the filter auto filterName = std::string("AlignFilter_") + std::to_string(numOfFilterLayers++); @@ -2054,32 +2050,25 @@ void MoveFakeQuantizeLayerIntoQuantParamsPass :: run() { return false; }; - auto allowFQFuse = [](CNNLayerPtr layer) -> bool { - auto doNotSkip = [](CNNLayerPtr layer) { - return false; + auto allowFQFuse = [this](CNNLayerPtr layer) -> bool { + auto skipNonFunctionalOrMemory = [](CNNLayerPtr layer) { + return LayerInfo(layer).isNonFunctional() || LayerInfo(layer).isMemory(); }; - - if (CNNNetGetAllNextLayersSkipCertain(layer, -1, doNotSkip).empty()) { - return false; - } - auto skipNonFunctional = [](CNNLayerPtr layer) { return LayerInfo(layer).isNonFunctional(); }; - - auto prevLayer = CNNNetPrevLayerSkipCertain(layer, 0, skipNonFunctional); - if (LayerInfo(prevLayer).isActivation() || LayerInfo(prevLayer).isConst() || LayerInfo(prevLayer).isMemory()) { - return true; + // Don't fuse FQ if it's the output layer for the network + if (CNNNetGetAllNextLayersSkipCertain(layer, -1, skipNonFunctionalOrMemory).empty()) { + return false; } - + // Fuse FQ if it's not required to change precision from int32 to int16 auto nextLayers = CNNNetGetAllNextLayersSkipCertain(layer, -1, skipNonFunctional); for (auto& l : nextLayers) { - if (!LayerInfo(l).isActivation()) { - return false; + if (getCandidatesForIdentityInsertion(l, getPassManager(), true).empty()) { + return true; } } - - return true; + return false; }; std::function propagateStatistics = @@ -2213,8 +2202,7 @@ void MoveFakeQuantizeLayerIntoQuantParamsPass :: run() { } // Allow FQ Fuse checks if FQ layer can be fused to a layer before or after. - // FQ Layer is fused only when previous layer is const, memory or activation layer - // or a next layer is activation layer. + // FQ Layer is fused if it's not required for precision change. bool isFQFuseAllowed = allowFQFuse(l); auto prevData = *prevDataIt; diff --git a/src/tests/functional/plugin/gna/scale_factors_tests/eltwise_act_fq.cpp b/src/tests/functional/plugin/gna/scale_factors_tests/eltwise_act_fq.cpp index 71602113d04..bbf6944212d 100644 --- a/src/tests/functional/plugin/gna/scale_factors_tests/eltwise_act_fq.cpp +++ b/src/tests/functional/plugin/gna/scale_factors_tests/eltwise_act_fq.cpp @@ -93,7 +93,7 @@ protected: const ngraph::Shape shape = {1, 128}; auto params = ngraph::builder::makeParams(ngPrc, {shape}); - auto lowNodeIn = ngraph::builder::makeConstant(ngPrc, {1}, { 100 * inputDataMin }); + auto lowNodeIn = ngraph::builder::makeConstant(ngPrc, {1}, { 100 * -inputDataMax }); auto highNodeIn = ngraph::builder::makeConstant(ngPrc, {1}, { 100 * inputDataMax }); auto fqIn = std::make_shared(params[0], lowNodeIn, highNodeIn, lowNodeIn, highNodeIn, levels16); diff --git a/src/tests/functional/plugin/gna/scale_factors_tests/matmul_overload_correction.cpp b/src/tests/functional/plugin/gna/scale_factors_tests/matmul_overload_correction.cpp index 073144fd09a..32ed52d44f1 100644 --- a/src/tests/functional/plugin/gna/scale_factors_tests/matmul_overload_correction.cpp +++ b/src/tests/functional/plugin/gna/scale_factors_tests/matmul_overload_correction.cpp @@ -67,6 +67,8 @@ protected: const ngraph::Shape shape2 = {1, inputShape[1] * inputShape[1]}; const float maxInputValue = 10.0f; auto params = ngraph::builder::makeParams(ngPrc, {shape1}); + auto relu = std::make_shared(params[0]); + std::shared_ptr input2; if (isSecondInputConst) { input2 = ngraph::builder::makeConstant(ngPrc, ngraph::Shape{shape1[1], shape1[1]}, @@ -78,7 +80,7 @@ protected: auto lowNodeIn1 = ngraph::builder::makeConstant(ngPrc, {1}, { -maxInputValue }); auto highNodeIn1 = ngraph::builder::makeConstant(ngPrc, {1}, { maxInputValue }); - auto fqIn1 = std::make_shared(params[0], lowNodeIn1, highNodeIn1, + auto fqIn1 = std::make_shared(relu, lowNodeIn1, highNodeIn1, lowNodeIn1, highNodeIn1, levels16); auto lowNodeIn2 = ngraph::builder::makeConstant(ngPrc, {1}, { -maxInputValue }); @@ -138,4 +140,4 @@ INSTANTIATE_TEST_SUITE_P(smoke_base, MatMulOverloadCorrectionTest, ::testing::ValuesIn({true, false}), ::testing::ValuesIn({true, false})), MatMulOverloadCorrectionTest::getTestCaseName); -} // namespace LayerTestsDefinitions \ No newline at end of file +} // namespace LayerTestsDefinitions diff --git a/src/tests/functional/plugin/gna/scale_factors_tests/weighable_layer_without_fq.cpp b/src/tests/functional/plugin/gna/scale_factors_tests/weighable_layer_without_fq.cpp index 96facb4bc4c..e1066d9b782 100644 --- a/src/tests/functional/plugin/gna/scale_factors_tests/weighable_layer_without_fq.cpp +++ b/src/tests/functional/plugin/gna/scale_factors_tests/weighable_layer_without_fq.cpp @@ -56,22 +56,23 @@ protected: auto ngPrc = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(netPrecision); auto params = ngraph::builder::makeParams(ngPrc, {inputShape}); + auto relu = std::make_shared(params[0]); auto fq1 = std::make_shared( - params[0], - ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {1.}), - ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {1.}), - ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {1.}), - ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {1.}), - 255); + relu, + ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {-10.}), + ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {10.}), + ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {-10.}), + ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {10.}), + static_cast(std::numeric_limits::max()) + 1); auto constant = ngraph::builder::makeConstant(ngPrc, constantShape, std::vector{}, true); auto fq2 = std::make_shared( constant, - ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {1}), - ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {1.}), - ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {1.}), - ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {1.}), - 255); - auto concat = ngraph::builder::makeConcat({fq1, fq2}, 0); + ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {-10}), + ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {10.}), + ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {-10.}), + ngraph::opset8::Constant::create(ngraph::element::f32, {1}, {10.}), + static_cast(std::numeric_limits::max()) + 1); + auto concat = ngraph::builder::makeConcat({fq1, fq2}, 1); function = std::make_shared(concat, params, "WeighableLayerWithoutFq"); } }; // class WeighableLayerWithoutFqTest @@ -91,7 +92,7 @@ const std::vector> inputShapes = { }; const std::vector> constantShapes = { - {{16, 5}} + {{1, 16}} }; const std::vector> configs = { diff --git a/src/tests/functional/shared_test_classes/src/subgraph/multiple_input_fq.cpp b/src/tests/functional/shared_test_classes/src/subgraph/multiple_input_fq.cpp index 367ceed7029..f63305f694c 100644 --- a/src/tests/functional/shared_test_classes/src/subgraph/multiple_input_fq.cpp +++ b/src/tests/functional/shared_test_classes/src/subgraph/multiple_input_fq.cpp @@ -27,15 +27,33 @@ void MultipleInputTest::SetUp() { std::tie(targetDevice, netPrecision, inputSize, config) = this->GetParam(); configuration.insert(config.begin(), config.end()); auto ngPrc = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(netPrecision); + + const float minInput = -10.0; + const float maxInput = 10.0; auto input = ngraph::builder::makeParams(ngPrc, {{1, inputSize}, {1, inputSize}, {1, inputSize}}); - auto fake1 = ngraph::builder::makeFakeQuantize(input[0], ngPrc, 255, { 1 }, { -0.5 }, { 0.5 }, { -0.5 }, { 0.5 }); - auto mul1 = ngraph::builder::makeEltwise(input[0], fake1, ngraph::helpers::EltwiseTypes::ADD); - auto fake2 = ngraph::builder::makeFakeQuantize(input[1], ngPrc, 255, { 1 }, { -0.5 }, { 0.5 }, { -0.5 }, { 0.5 }); - auto mul2 = ngraph::builder::makeEltwise(input[1], fake2, ngraph::helpers::EltwiseTypes::ADD); - auto mul3 = ngraph::builder::makeEltwise(mul1, mul2, ngraph::helpers::EltwiseTypes::ADD); - auto fake3 = ngraph::builder::makeFakeQuantize(input[2], ngPrc, 255, { 1 }, { -0.5 }, { 0.5 }, { -0.5 }, { 0.5 }); - auto mul4 = ngraph::builder::makeEltwise(fake3, mul3, ngraph::helpers::EltwiseTypes::ADD); - auto result = std::make_shared(mul4); + auto fake1 = ngraph::builder::makeFakeQuantize(input[0], ngPrc, std::numeric_limits::max(), { 1 }, + { minInput }, { maxInput }, { minInput }, { maxInput }); + auto add1 = ngraph::builder::makeEltwise(input[0], fake1, ngraph::helpers::EltwiseTypes::ADD); + auto fake_add1 = ngraph::builder::makeFakeQuantize(add1, ngPrc, std::numeric_limits::max(), { 1 }, + { 2 * minInput }, { 2 * maxInput }, { 2 * minInput }, { 2 * maxInput }); + + auto fake2 = ngraph::builder::makeFakeQuantize(input[1], ngPrc, std::numeric_limits::max(), { 1 }, + { minInput }, { maxInput }, { minInput }, { maxInput }); + auto add2 = ngraph::builder::makeEltwise(input[1], fake2, ngraph::helpers::EltwiseTypes::ADD); + auto fake_add2 = ngraph::builder::makeFakeQuantize(add2, ngPrc, std::numeric_limits::max(), { 1 }, + { 2 * minInput }, { 2 * maxInput }, { 2 * minInput }, { 2 * maxInput }); + + auto add3 = ngraph::builder::makeEltwise(fake_add1, fake_add2, ngraph::helpers::EltwiseTypes::ADD); + auto fake_add3 = ngraph::builder::makeFakeQuantize(add3, ngPrc, std::numeric_limits::max(), { 1 }, + { 4 * minInput }, { 4 * maxInput }, { 4 * minInput }, { 4 * maxInput }); + + auto fake3 = ngraph::builder::makeFakeQuantize(input[2], ngPrc, std::numeric_limits::max(), { 1 }, + { minInput }, { maxInput }, { minInput }, { maxInput }); + auto add4 = ngraph::builder::makeEltwise(fake3, fake_add3, ngraph::helpers::EltwiseTypes::ADD); + auto fake_add4 = ngraph::builder::makeFakeQuantize(add4, ngPrc, std::numeric_limits::max(), { 1 }, + { 5 * minInput }, { 5 * maxInput }, { 5 * minInput }, { 5 * maxInput }); + + auto result = std::make_shared(fake_add4); function = std::make_shared(ngraph::ResultVector{result}, input, "multiple_input"); } From a9853d2790cb65a3cc55f7a1f2697e158537189f Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 31 Mar 2022 13:56:25 +0300 Subject: [PATCH 27/39] [GNA] Additional tests on compact mode (#10969) * Moved InitGNADevice to plugin constructor * Added tests for ordering layers * Added allocator header * Fixed fused_iterator header * protected GNAMemRequestsQueue properties * Fixed unit test names * Fixed compile issue * Fixed default initialization * Fixed depricated matchers * Fixed pwl deprecated tests * Added page alignment * Reset gnadevice in the tests * Update src/plugins/intel_gna/gna_fused_iterator.hpp Co-authored-by: Nadezhda Ageeva * Revert "Update src/plugins/intel_gna/gna_fused_iterator.hpp" This reverts commit d624bdadafa4059b74d3068bd677451266b9f6b3. Co-authored-by: Nadezhda Ageeva --- .../intel_gna/gna2_model_export_helper.cpp | 4 +- src/plugins/intel_gna/gna_device.cpp | 4 +- src/plugins/intel_gna/gna_fused_iterator.hpp | 1 + src/plugins/intel_gna/gna_plugin.cpp | 38 ++-- src/plugins/intel_gna/memory/gna_memory.hpp | 2 + .../unit/gna/gna_memory_compact_test.cpp | 117 +++++++++++ .../unit/engines/gna/gna_matcher.cpp | 198 +++++++++--------- .../unit/engines/gna/gna_matcher.hpp | 1 + .../unit/engines/gna/gna_pwl_test.cpp | 40 ++-- 9 files changed, 260 insertions(+), 145 deletions(-) diff --git a/src/plugins/intel_gna/gna2_model_export_helper.cpp b/src/plugins/intel_gna/gna2_model_export_helper.cpp index 66fcc2d7062..af7e0489bfe 100644 --- a/src/plugins/intel_gna/gna2_model_export_helper.cpp +++ b/src/plugins/intel_gna/gna2_model_export_helper.cpp @@ -27,7 +27,7 @@ void * ExportSueLegacyUsingGnaApi2( status = Gna2ModelExportConfigSetTarget(exportConfig, Gna2DeviceVersionEmbedded1_0); GNADeviceHelper::checkGna2Status(status, "Gna2ModelExportConfigSetTarget"); - void * bufferSueCreekHeader; + void * bufferSueCreekHeader = nullptr; uint32_t bufferSueCreekHeaderSize; status = Gna2ModelExport(exportConfig, @@ -37,7 +37,7 @@ void * ExportSueLegacyUsingGnaApi2( (*modelHeader) = *(reinterpret_cast(bufferSueCreekHeader)); - void * bufferDump; + void * bufferDump = nullptr; uint32_t bufferDumpSize; status = Gna2ModelExport(exportConfig, Gna2ModelExportComponentLegacySueCreekDump, diff --git a/src/plugins/intel_gna/gna_device.cpp b/src/plugins/intel_gna/gna_device.cpp index 64041aa7e49..8699de95e54 100644 --- a/src/plugins/intel_gna/gna_device.cpp +++ b/src/plugins/intel_gna/gna_device.cpp @@ -111,7 +111,7 @@ void GNADeviceHelper::enforceLegacyCnnsWhenNeeded(Gna2Model& gnaModel) { uint32_t GNADeviceHelper::createModel(Gna2Model& gnaModel) const { std::unique_lock lockGnaCalls{ acrossPluginsSync }; - uint32_t modelId; + uint32_t modelId = 0; const auto legacyExecTarget = enforceLegacyCnnNeeded(); if (legacyExecTarget) { enforceLegacyCnns(gnaModel); @@ -231,7 +231,7 @@ void GNADeviceHelper::checkGna2Status(Gna2Status status, const Gna2Model& gnaMod decoratedGnaLibVersion(); } - Gna2ModelError error; + Gna2ModelError error{}; auto getLastErrorStatus = Gna2ModelGetLastError(&error); checkGna2Status(getLastErrorStatus, "Gna2ModelGetLastError"); diff --git a/src/plugins/intel_gna/gna_fused_iterator.hpp b/src/plugins/intel_gna/gna_fused_iterator.hpp index 64cedbe2a52..3764eca818b 100644 --- a/src/plugins/intel_gna/gna_fused_iterator.hpp +++ b/src/plugins/intel_gna/gna_fused_iterator.hpp @@ -8,6 +8,7 @@ #include #include #include "gna_graph_tools.hpp" +#include "layers/gna_layer_helpers.hpp" namespace GNAPluginNS { /** diff --git a/src/plugins/intel_gna/gna_plugin.cpp b/src/plugins/intel_gna/gna_plugin.cpp index 9b168b22bc7..deec8577ce4 100644 --- a/src/plugins/intel_gna/gna_plugin.cpp +++ b/src/plugins/intel_gna/gna_plugin.cpp @@ -31,7 +31,6 @@ #include "frontend/model_quantizer.hpp" #include "gna_fused_iterator.hpp" #include "backend/am_intel_dnn.hpp" -#include "memory/gna_allocator.hpp" #include "memory/gna_memory_state.hpp" #include "gna_model_serial.hpp" #include "runtime/gna_float_runtime.hpp" @@ -324,11 +323,13 @@ void GNAPlugin::ImportFrames(void *ptr_dst, GNAPlugin::GNAPlugin() { Init(); UpdateFieldsFromConfig(); + InitGNADevice(); } GNAPlugin::GNAPlugin(const std::map& configMap) { Init(); SetConfig(configMap); + InitGNADevice(); } void GNAPlugin::Init() { @@ -345,14 +346,18 @@ void GNAPlugin::Init() { void GNAPlugin::InitGNADevice() { OV_ITT_SCOPED_TASK(itt::domains::GNA_LT, "InitGNADevice"); - gnadevice = std::make_shared(config.gnaExecTarget, - config.gnaCompileTarget, - config.swExactMode, - gnaFlags->performance_counting, - !config.dumpXNNPath.empty(), - GetDeviceVersionFromString(config.dumpXNNGeneration)); - size_t page_size_bytes = 4096; - gnamem = std::make_shared(memory::make_polymorph(gnadevice), page_size_bytes); + if (gnaFlags->sw_fp32) { + gnamem.reset(new gna_memory_type(memory::make_polymorph>())); + } else { + gnadevice = std::make_shared(config.gnaExecTarget, + config.gnaCompileTarget, + config.swExactMode, + gnaFlags->performance_counting, + !config.dumpXNNPath.empty(), + GetDeviceVersionFromString(config.dumpXNNGeneration)); + size_t page_size_bytes = 4096; + gnamem = std::make_shared(memory::make_polymorph(gnadevice), page_size_bytes); + } graphCompiler.setGNAMemoryPtr(gnamem); } @@ -635,10 +640,6 @@ void GNAPlugin::LoadNetwork(CNNNetwork & _network) { OV_ITT_SCOPED_TASK(itt::domains::GNAPlugin, "LoadNetwork"); std::shared_ptr convertedNetwork; - if (!gnaFlags->sw_fp32) { - InitGNADevice(); - } - std::string effectiveGnaCompileTarget = config.gnaCompileTarget; if (gnadevice) { effectiveGnaCompileTarget = gnadevice->getEffectiveGnaCompileTarget(); @@ -894,15 +895,7 @@ void GNAPlugin::LoadNetwork(CNNNetwork & _network) { graphCompiler.fillMemoryConnections(memoryPairs); if (!graphCompiler.memory_connection.empty() && gnaFlags->num_requests != 1) { - // TODO: check if updating the number of threads is needed for sw_fp32 gnaFlags->num_requests = 1; - if (!gnaFlags->sw_fp32) - InitGNADevice(); - } - - if (gnaFlags->sw_fp32) { - gnamem.reset(new gna_memory_type(memory::make_polymorph>())); - graphCompiler.setGNAMemoryPtr(gnamem); } // keep inputs information and create input primitives @@ -1508,9 +1501,6 @@ void GNAPlugin::SetName(const std::string & pluginName) noexcept { InferenceEngine::IExecutableNetworkInternal::Ptr GNAPlugin::ImportNetwork(std::istream& networkModel) { auto header = GNAModelSerial::ReadHeader(networkModel); - InitGNADevice(); - - graphCompiler.setGNAMemoryPtr(gnamem); void *basePtr = nullptr; gnamem->reserve_ptr(nullptr, &basePtr, header.gnaMemSize); gnamem->commit(); diff --git a/src/plugins/intel_gna/memory/gna_memory.hpp b/src/plugins/intel_gna/memory/gna_memory.hpp index beea39cf70f..6213a507515 100644 --- a/src/plugins/intel_gna/memory/gna_memory.hpp +++ b/src/plugins/intel_gna/memory/gna_memory.hpp @@ -17,6 +17,7 @@ #include "gna_lib_ver_selector.hpp" #include "memory_solver.hpp" #include "gna_plugin_log.hpp" +#include "memory/gna_allocator.hpp" #ifdef GNA_HEAP_PROFILER #include @@ -32,6 +33,7 @@ namespace memory { */ template> class GNAMemory : public GNAMemRequestsQueue { +protected: std::vector _future_heap; std::list> _local_storage; size_t _total = 0; diff --git a/src/tests/unit/gna/gna_memory_compact_test.cpp b/src/tests/unit/gna/gna_memory_compact_test.cpp index 5f426005977..73ff7e022ac 100644 --- a/src/tests/unit/gna/gna_memory_compact_test.cpp +++ b/src/tests/unit/gna/gna_memory_compact_test.cpp @@ -3,9 +3,18 @@ // #include +#include + #include #include +#include +#include +#include "ngraph_functions/builders.hpp" #include "memory/gna_memory.hpp" +#include "gna_plugin.hpp" +#include "gna_fused_iterator.hpp" +#include "gna_data_types.hpp" + using namespace InferenceEngine; using namespace GNAPluginNS::memory; @@ -247,4 +256,112 @@ TEST_F(GNAMemoryCompactTest, canOptimizeReservePtrWithOffset) { mem.commit(isCompact); ASSERT_EQ(mem.getRWBytes(), 4 * sizeof(float)); ASSERT_EQ(mem.getTotalBytes(), 4 * sizeof(float)); +} + +class GNAMemoryTested : public GNAPluginNS::memory::GNAMemory> { +using GNAMemory::GNAMemory; + +public: + void Test() { + // filtering RW allocation requests only + auto filter_req = [] (const MemRequest &re) { return re._region == REGION_RW && re._type != REQUEST_BIND; }; + std::vector test_reqs; + auto it = std::copy_if(_future_heap.begin(), _future_heap.end(), std::back_inserter(test_reqs), filter_req); + + // intercrossing condition + auto is_crossed = [] (const MemRequest &re1, const MemRequest &re2) { + const std::pair limits_default {0, UINT16_MAX}; + if (re1._life_limits == limits_default || re2._life_limits == limits_default) { + return true; + } + return (re1._life_limits.first > re2._life_limits.first && re1._life_limits.first < re2._life_limits.second) || + (re1._life_limits.second > re2._life_limits.first && re1._life_limits.second < re2._life_limits.second); + }; + + // verify that requests are intercrossed + for (auto re_it_1 = test_reqs.begin(); re_it_1 != test_reqs.end(); ++re_it_1) { + for (auto re_it_2 = re_it_1 + 1; re_it_2 != test_reqs.end(); ++re_it_2) { + ASSERT_TRUE(is_crossed(*re_it_1, *re_it_2)); + } + } + } +}; + +class GNAPluginTested : public GNAPluginNS::GNAPlugin { +public: + std::shared_ptr gnamem_t; + GNAPluginTested() : GNAPluginNS::GNAPlugin() { + gnamem_t = std::make_shared(make_polymorph>()); + gnamem = gnamem_t; + graphCompiler.setGNAMemoryPtr(gnamem); + gnadevice.reset(); + } + void Test() { + gnamem_t->Test(); + } +}; + +class GNAMemoryOrderTest : public ::testing::Test {}; + +TEST_F(GNAMemoryOrderTest, orderingFusedLayersActivation) { + auto plugin = GNAPluginTested(); + + ov::Shape input_shape = { 1, 8, 20, 16 }; + ov::Strides strides = { 1, 1 }; + ov::Strides dilations = { 1, 1 }; + ov::CoordinateDiff pad_begin(0, 0), pad_end(0, 0); + auto weights = ngraph::builder::makeConstant(ov::element::f32, { 8, 8, 1, 1 }, { 1.f }); + + auto input = std::make_shared(ov::element::f32, input_shape); + auto conv = std::make_shared(input, weights, strides, pad_begin, pad_end, dilations); + auto activation = ngraph::builder::makeActivation(conv, ov::element::f32, ngraph::helpers::ActivationTypes::Sigmoid); + auto result = std::make_shared(activation); + auto function = std::make_shared(ov::ResultVector({result}), ov::ParameterVector({input}), "convolution"); + + InferenceEngine::CNNNetwork cnn_network(function); + plugin.LoadNetwork(cnn_network); + plugin.Test(); +} + +TEST_F(GNAMemoryOrderTest, orderingFusedLayersMaxPool) { + auto plugin = GNAPluginTested(); + + ov::Shape input_shape = { 1, 8, 20, 16 }; + ov::Strides strides = { 1, 1 }; + ov::Strides dilations = { 1, 1 }; + ov::CoordinateDiff pad_begin(0, 0), pad_end(0, 0); + auto weights = ngraph::builder::makeConstant(ov::element::f32, { 8, 8, 1, 1 }, { 1.f }); + + auto input = std::make_shared(ov::element::f32, input_shape); + auto conv = std::make_shared(input, weights, strides, pad_begin, pad_end, dilations); + auto maxpool = ngraph::builder::makePooling(conv, {1, 1}, {0, 0}, {0, 0}, {1, 1}, ngraph::op::RoundingType::FLOOR, + ngraph::op::PadType::VALID, false, ngraph::helpers::PoolingTypes::MAX); + auto result = std::make_shared(maxpool); + auto function = std::make_shared(ov::ResultVector({result}), ov::ParameterVector({input}), "convolution"); + + InferenceEngine::CNNNetwork cnn_network(function); + plugin.LoadNetwork(cnn_network); + plugin.Test(); +} + +TEST_F(GNAMemoryOrderTest, orderingFusedLayersActivationMaxPool) { + auto plugin = GNAPluginTested(); + + ov::Shape input_shape = { 1, 8, 20, 16 }; + ov::Strides strides = { 1, 1 }; + ov::Strides dilations = { 1, 1 }; + ov::CoordinateDiff pad_begin(0, 0), pad_end(0, 0); + auto weights = ngraph::builder::makeConstant(ov::element::f32, { 8, 8, 1, 1 }, { 1.f }); + + auto input = std::make_shared(ov::element::f32, input_shape); + auto conv = std::make_shared(input, weights, strides, pad_begin, pad_end, dilations); + auto activation = ngraph::builder::makeActivation(conv, ov::element::f32, ngraph::helpers::ActivationTypes::Sigmoid); + auto maxpool = ngraph::builder::makePooling(activation, {1, 1}, {0, 0}, {0, 0}, {1, 1}, ngraph::op::RoundingType::FLOOR, + ngraph::op::PadType::VALID, false, ngraph::helpers::PoolingTypes::MAX); + auto result = std::make_shared(maxpool); + auto function = std::make_shared(ov::ResultVector({result}), ov::ParameterVector({input}), "convolution"); + + InferenceEngine::CNNNetwork cnn_network(function); + plugin.LoadNetwork(cnn_network); + plugin.Test(); } \ No newline at end of file diff --git a/src/tests_deprecated/unit/engines/gna/gna_matcher.cpp b/src/tests_deprecated/unit/engines/gna/gna_matcher.cpp index 3ab503f133f..8498cfca9de 100644 --- a/src/tests_deprecated/unit/engines/gna/gna_matcher.cpp +++ b/src/tests_deprecated/unit/engines/gna/gna_matcher.cpp @@ -101,12 +101,112 @@ void expect_enqueue_calls(GNACppApi &mockApi){ void GNAPropagateMatcher :: match() { try { // matching gna propagate forward call. - GNAPlugin plugin(_env.config); + size_t inputSize = 10; size_t outputSize = 10; InputsDataMap inputsInfo; OutputsDataMap outputsInfo; + StrictMock mockApi; + std::vector data; + + if (_env.config[GNA_CONFIG_KEY(DEVICE_MODE)].compare(GNA_CONFIG_VALUE(SW_FP32)) != 0 && + !_env.matchThrows) { + EXPECT_CALL(mockApi, Gna2MemoryAlloc(_, _, _)).WillOnce(Invoke([&data]( + uint32_t sizeRequested, + uint32_t *sizeGranted, + void **memoryAddress + ) { + data.resize(sizeRequested); + *sizeGranted = sizeRequested; + *memoryAddress = &data.front(); + return Gna2StatusSuccess; + })); + + EXPECT_CALL(mockApi, Gna2DeviceGetVersion(_,_)).WillOnce(Invoke([]( + uint32_t deviceIndex, + enum Gna2DeviceVersion * deviceVersion) { + *deviceVersion = Gna2DeviceVersionSoftwareEmulation; + return Gna2StatusSuccess; + })); + + EXPECT_CALL(mockApi, Gna2DeviceOpen(_)).WillOnce(Return(Gna2StatusSuccess)); + + EXPECT_CALL(mockApi, Gna2GetLibraryVersion(_,_)).Times(AtLeast(0)).WillRepeatedly(Return(Gna2StatusSuccess)); + + EXPECT_CALL(mockApi, Gna2InstrumentationConfigCreate(_,_,_,_)).WillOnce(Return(Gna2StatusSuccess)); + + if(_env.is_setup_of_omp_theads_expected == true) { + EXPECT_CALL(mockApi, Gna2DeviceSetNumberOfThreads(_,_)).WillOnce(Return(Gna2StatusSuccess)); + } + + std::unique_ptr combined(new NNetComponentMatcher()); + + for (auto & matchWhat : _env.whatToMatch) { + switch(matchWhat.type) { + case GnaPluginTestEnvironment::matchPrecision : + combined->add(new NNetPrecisionMatcher(_env.nnet_precision, INTEL_AFFINE)); + break; + case GnaPluginTestEnvironment::matchProcType : + expect_enqueue_calls(mockApi); + break; + case GnaPluginTestEnvironment::matchPwlInserted : + combined->add(new PWLMatcher(_env.matchInserted, matchWhat.matchQuantity, _env.pwlsToMatchWith)); + break; + case GnaPluginTestEnvironment::matchConvInserted: + combined->add(new ConvoluionLayerMatcher(_env.matchInserted, matchWhat.matchQuantity)); + break; + case GnaPluginTestEnvironment::matchMaxPoolingInserted: + combined->add(new PoolingLayerMatcher(_env.matchInserted, matchWhat.matchQuantity, true)); + break; + case GnaPluginTestEnvironment::matchPwlQuantizeMetrics : + combined->add(new PWLQuantizationMetricsMatcher(_env.type, + _env.quantization_presicion_threshold, + _env.quantization_segments_threshold)); + break; + case GnaPluginTestEnvironment::matchCopyInserted : + combined->add(new CopyLayerMatcher(_env.matchInserted, matchWhat.matchQuantity)); + break; + case GnaPluginTestEnvironment::matchDiagonalInserted : + combined->add(new DiagLayerMatcher(_env.matchInserted, matchWhat.matchQuantity)); + break; + case GnaPluginTestEnvironment::saveArgs : + expect_enqueue_calls(mockApi); + break; + case GnaPluginTestEnvironment::matchInputData : + combined->add(new InputDataMatcher(_env.input_processed)); + break; + case GnaPluginTestEnvironment::fillOutputValues : + combined->add(new OutputFiller(_env.fillValue, _env.fillValue)); + break; + case GnaPluginTestEnvironment::matchAffineWeightsTranspose: + HasWeightsTranspozed(combined, _env.transposedData, _env.transposeArgs); + break; + case GnaPluginTestEnvironment::matchAffineWeights: + HasWeightsEq(combined, _env.transposedData); + break; + case GnaPluginTestEnvironment::matchAffineWeightsSize: + HasWeightsSizeEq(combined, _env.matched_weight_size); + break; + case GnaPluginTestEnvironment::saveAffineWeights: + SaveWeights(combined, _env.transposedData, _env.transposedArgsForSaving); + break; + default: + expect_enqueue_calls(mockApi); + break; + } + } + if (combined && !combined->empty()) { + expect_enqueue_calls(mockApi); + } + + EXPECT_CALL(mockApi, Gna2MemoryFree(_)).WillOnce(Return(Gna2StatusSuccess)); + + EXPECT_CALL(mockApi, Gna2DeviceClose(_)).WillOnce(Return(Gna2StatusSuccess)); + } + + GNAPlugin plugin(_env.config); + auto loadNetworkFromIR = [&] () -> InferenceEngine::CNNNetwork { Core core; auto weights_fake = make_shared_blob(TensorDesc(Precision::U8, @@ -273,102 +373,6 @@ void GNAPropagateMatcher :: match() { } }; - - StrictMock mockApi; - std::vector data; - - if (_env.config[GNA_CONFIG_KEY(DEVICE_MODE)].compare(GNA_CONFIG_VALUE(SW_FP32)) != 0 && - !_env.matchThrows) { - EXPECT_CALL(mockApi, Gna2MemoryAlloc(_, _, _)).WillOnce(Invoke([&data]( - uint32_t sizeRequested, - uint32_t *sizeGranted, - void **memoryAddress - ) { - data.resize(sizeRequested); - *sizeGranted = sizeRequested; - *memoryAddress = &data.front(); - return Gna2StatusSuccess; - })); - - EXPECT_CALL(mockApi, Gna2DeviceGetVersion(_,_)).WillOnce(Invoke([]( - uint32_t deviceIndex, - enum Gna2DeviceVersion * deviceVersion) { - *deviceVersion = Gna2DeviceVersionSoftwareEmulation; - return Gna2StatusSuccess; - })); - - EXPECT_CALL(mockApi, Gna2DeviceOpen(_)).WillOnce(Return(Gna2StatusSuccess)); - - EXPECT_CALL(mockApi, Gna2GetLibraryVersion(_,_)).Times(AtLeast(0)).WillRepeatedly(Return(Gna2StatusSuccess)); - - EXPECT_CALL(mockApi, Gna2InstrumentationConfigCreate(_,_,_,_)).WillOnce(Return(Gna2StatusSuccess)); - - - - if(_env.is_setup_of_omp_theads_expected == true) { - EXPECT_CALL(mockApi, Gna2DeviceSetNumberOfThreads(_,_)).WillOnce(Return(Gna2StatusSuccess)); - } - std::unique_ptr combined(new NNetComponentMatcher()); - - for (auto & matchWhat : _env.whatToMatch) { - switch(matchWhat.type) { - case GnaPluginTestEnvironment::matchPrecision : - combined->add(new NNetPrecisionMatcher(_env.nnet_precision, INTEL_AFFINE)); - break; - case GnaPluginTestEnvironment::matchProcType : - expect_enqueue_calls(mockApi); - break; - case GnaPluginTestEnvironment::matchPwlInserted : - combined->add(new PWLMatcher(_env.matchInserted, matchWhat.matchQuantity, _env.pwlsToMatchWith)); - break; - case GnaPluginTestEnvironment::matchConvInserted: - combined->add(new ConvoluionLayerMatcher(_env.matchInserted, matchWhat.matchQuantity)); - break; - case GnaPluginTestEnvironment::matchMaxPoolingInserted: - combined->add(new PoolingLayerMatcher(_env.matchInserted, matchWhat.matchQuantity, true)); - break; - case GnaPluginTestEnvironment::matchPwlQuantizeMetrics : - combined->add(new PWLQuantizationMetricsMatcher(_env.type, - _env.quantization_presicion_threshold, - _env.quantization_segments_threshold)); - break; - case GnaPluginTestEnvironment::matchCopyInserted : - combined->add(new CopyLayerMatcher(_env.matchInserted, matchWhat.matchQuantity)); - break; - case GnaPluginTestEnvironment::matchDiagonalInserted : - combined->add(new DiagLayerMatcher(_env.matchInserted, matchWhat.matchQuantity)); - break; - case GnaPluginTestEnvironment::saveArgs : - expect_enqueue_calls(mockApi); - break; - case GnaPluginTestEnvironment::matchInputData : - combined->add(new InputDataMatcher(_env.input_processed)); - break; - case GnaPluginTestEnvironment::fillOutputValues : - combined->add(new OutputFiller(_env.fillValue, _env.fillValue)); - break; - case GnaPluginTestEnvironment::matchAffineWeightsTranspose: - HasWeightsTranspozed(combined, _env.transposedData, _env.transposeArgs); - break; - case GnaPluginTestEnvironment::matchAffineWeights: - HasWeightsEq(combined, _env.transposedData); - break; - case GnaPluginTestEnvironment::matchAffineWeightsSize: - HasWeightsSizeEq(combined, _env.matched_weight_size); - break; - case GnaPluginTestEnvironment::saveAffineWeights: - SaveWeights(combined, _env.transposedData, _env.transposedArgsForSaving); - break; - default: - expect_enqueue_calls(mockApi); - break; - } - } - if (combined && !combined->empty()) { - expect_enqueue_calls(mockApi); - } - } - loadNetwork(); if (!inputsInfo.empty()) { diff --git a/src/tests_deprecated/unit/engines/gna/gna_matcher.hpp b/src/tests_deprecated/unit/engines/gna/gna_matcher.hpp index 21f60a6a373..e37c592d1a7 100644 --- a/src/tests_deprecated/unit/engines/gna/gna_matcher.hpp +++ b/src/tests_deprecated/unit/engines/gna/gna_matcher.hpp @@ -642,6 +642,7 @@ class GNATest : public U, public GNATestConfigurability> { return *this; } GNATest & gna() { + _env.config[GNA_CONFIG_KEY(DEVICE_MODE)] = GNA_CONFIG_VALUE(AUTO); return *this; } GNATest & from() { diff --git a/src/tests_deprecated/unit/engines/gna/gna_pwl_test.cpp b/src/tests_deprecated/unit/engines/gna/gna_pwl_test.cpp index 8e98660b526..c14bdf9a01c 100644 --- a/src/tests_deprecated/unit/engines/gna/gna_pwl_test.cpp +++ b/src/tests_deprecated/unit/engines/gna/gna_pwl_test.cpp @@ -41,7 +41,7 @@ using namespace GNATestIRs; TEST_F(PWLAproximationTest, forTanhOnRecursiveAlgoWithPrecisionThresholdIsSuccess) { assert_that().onInferNgraphModel(detail::CreateActivationFunction({1, 10})) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .propagate_forward() .called_with() @@ -51,7 +51,7 @@ TEST_F(PWLAproximationTest, forTanhOnRecursiveAlgoWithPrecisionThresholdIsSucces TEST_F(PWLAproximationTest, forSigmoidOnRecursiveAlgoWithPrecisionThresholdIsSuccess) { assert_that().onInferNgraphModel(detail::CreateActivationFunction({1, 10})) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .propagate_forward() .called_with() @@ -61,7 +61,7 @@ TEST_F(PWLAproximationTest, forSigmoidOnRecursiveAlgoWithPrecisionThresholdIsSuc TEST_F(PWLAproximationTest, forReLUonRecursiveAlgoWithPrecisionThresholdIsSuccess) { assert_that().onInferNgraphModel(detail::CreateActivationFunction({1, 1, 10, 10})) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .propagate_forward() .called_with() @@ -71,7 +71,7 @@ TEST_F(PWLAproximationTest, forReLUonRecursiveAlgoWithPrecisionThresholdIsSucces TEST_F(PWLAproximationTest, forLeakyReLUonRecursiveAlgoWithPrecisionThresholdIsSuccess) { assert_that().onInferModel(LeakyReLUActivationModel()) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .propagate_forward() .called_with() @@ -81,7 +81,7 @@ TEST_F(PWLAproximationTest, forLeakyReLUonRecursiveAlgoWithPrecisionThresholdIsS TEST_F(PWLAproximationTest, DISABLED_forIdentityOnRecursiveAlgoWithPrecisionThresholdIsSuccess) { assert_that().onInferModel(IdentityActivationModel()) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .propagate_forward() .called_with() @@ -91,7 +91,7 @@ TEST_F(PWLAproximationTest, DISABLED_forIdentityOnRecursiveAlgoWithPrecisionThre TEST_F(PWLAproximationTest, forClampOnRecursiveAlgoWithPrecisionThresholdIsSuccess) { assert_that().onInferNgraphModel(detail::CreateActivationFunction({1, 10}, -50, 50)) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .propagate_forward() .called_with() @@ -101,7 +101,7 @@ TEST_F(PWLAproximationTest, forClampOnRecursiveAlgoWithPrecisionThresholdIsSucce TEST_F(PWLAproximationTest, forTanhOnUniformAlgoWithPrecisionThresholdIsSuccess) { assert_that().onInferNgraphModel(detail::CreateActivationFunction({1, 10})) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .withUniformPWLAlgo() .propagate_forward() @@ -112,7 +112,7 @@ TEST_F(PWLAproximationTest, forTanhOnUniformAlgoWithPrecisionThresholdIsSuccess) TEST_F(PWLAproximationTest, forSigmoidOnUniformAlgoWithPrecisionThresholdIsSuccess) { assert_that().onInferNgraphModel(detail::CreateActivationFunction({1, 10})) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .withUniformPWLAlgo() .propagate_forward() @@ -123,7 +123,7 @@ TEST_F(PWLAproximationTest, forSigmoidOnUniformAlgoWithPrecisionThresholdIsSucce TEST_F(PWLAproximationTest, DISABLED_forIdentityOnUniformAlgoWithPrecisionThresholdIsSuccess) { assert_that().onInferModel(IdentityActivationModel()) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .withUniformPWLAlgo() .propagate_forward() @@ -134,7 +134,7 @@ TEST_F(PWLAproximationTest, DISABLED_forIdentityOnUniformAlgoWithPrecisionThresh TEST_F(PWLAproximationTest, forClampOnUniformAlgoWithPrecisionThresholdIsSuccess) { assert_that().onInferNgraphModel(detail::CreateActivationFunction({1, 10}, -50, 50)) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .withUniformPWLAlgo() .propagate_forward() @@ -145,7 +145,7 @@ TEST_F(PWLAproximationTest, forClampOnUniformAlgoWithPrecisionThresholdIsSuccess TEST_F(PWLAproximationTest, forSigmoidonRecursiveAlgoWithSegmentThresholdIsSuccess) { assert_that().onInferNgraphModel(detail::CreateActivationFunction({1, 10})) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .propagate_forward() .called_with() @@ -155,7 +155,7 @@ TEST_F(PWLAproximationTest, forSigmoidonRecursiveAlgoWithSegmentThresholdIsSucce TEST_F(PWLAproximationTest, forTanhonRecursiveAlgoWithSegmentThresholdIsSuccess) { assert_that().onInferNgraphModel(detail::CreateActivationFunction({1, 10})) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .propagate_forward() .called_with() @@ -165,7 +165,7 @@ TEST_F(PWLAproximationTest, forTanhonRecursiveAlgoWithSegmentThresholdIsSuccess) TEST_F(PWLAproximationTest, forReLUonRecursiveAlgoWithSegmentThresholdIsSuccess) { assert_that().onInferNgraphModel(detail::CreateActivationFunction({1, 1, 10, 10})) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .propagate_forward() .called_with() @@ -175,7 +175,7 @@ TEST_F(PWLAproximationTest, forReLUonRecursiveAlgoWithSegmentThresholdIsSuccess) TEST_F(PWLAproximationTest, forLeakyReLUonRecursiveAlgoWithSegmentThresholdIsSuccess) { assert_that().onInferModel(LeakyReLUActivationModel()) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .propagate_forward() .called_with() @@ -185,7 +185,7 @@ TEST_F(PWLAproximationTest, forLeakyReLUonRecursiveAlgoWithSegmentThresholdIsSuc TEST_F(PWLAproximationTest, DISABLED_forIdentityOnRecursiveAlgoWithSegmentThresholdIsSuccess) { assert_that().onInferModel(IdentityActivationModel()) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .propagate_forward() .called_with() @@ -195,7 +195,7 @@ TEST_F(PWLAproximationTest, DISABLED_forIdentityOnRecursiveAlgoWithSegmentThresh TEST_F(PWLAproximationTest, forClampOnRecursiveAlgoWithSegmentThresholdIsSuccess) { assert_that().onInferNgraphModel(detail::CreateActivationFunction({1, 10}, -50, 50)) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .propagate_forward() .called_with() @@ -205,7 +205,7 @@ TEST_F(PWLAproximationTest, forClampOnRecursiveAlgoWithSegmentThresholdIsSuccess TEST_F(PWLAproximationTest, forSigmoidonUniformAlgoWithSegmentThresholdIsSuccess) { assert_that().onInferNgraphModel(detail::CreateActivationFunction({1, 10})) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .withUniformPWLAlgo() .propagate_forward() @@ -216,7 +216,7 @@ TEST_F(PWLAproximationTest, forSigmoidonUniformAlgoWithSegmentThresholdIsSuccess TEST_F(PWLAproximationTest, forTanhonUniformAlgoWithSegmentThresholdIsSuccess) { assert_that().onInferNgraphModel(detail::CreateActivationFunction({1, 10})) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .withUniformPWLAlgo() .propagate_forward() @@ -227,7 +227,7 @@ TEST_F(PWLAproximationTest, forTanhonUniformAlgoWithSegmentThresholdIsSuccess) { TEST_F(PWLAproximationTest, DISABLED_forIdentityOnUniformAlgoWithSegmentThresholdIsSuccess) { assert_that().onInferModel(IdentityActivationModel()) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .withUniformPWLAlgo() .propagate_forward() @@ -238,7 +238,7 @@ TEST_F(PWLAproximationTest, DISABLED_forIdentityOnUniformAlgoWithSegmentThreshol TEST_F(PWLAproximationTest, forClampOnUniformAlgoWithSegmentThresholdIsSuccess) { assert_that().onInferNgraphModel(detail::CreateActivationFunction({1, 10}, -50, 50)) - .inNotCompactMode() + .gna() .withGNAConfig(GNA_CONFIG_KEY(SCALE_FACTOR), 1.0f) .withUniformPWLAlgo() .propagate_forward() From 763d52275957d73934269c433fbd4348c0a4b422 Mon Sep 17 00:00:00 2001 From: Smirnov Grigorii Date: Thu, 31 Mar 2022 14:06:36 +0300 Subject: [PATCH 28/39] add graph_comparator tests (#11360) --- .../common_test_utils/CMakeLists.txt | 3 + .../common_test_utils/tests/CMakeLists.txt | 11 + .../tests/graph_comparator_tests.cpp | 622 ++++++++++++++++++ 3 files changed, 636 insertions(+) create mode 100644 src/tests/ie_test_utils/common_test_utils/tests/CMakeLists.txt create mode 100644 src/tests/ie_test_utils/common_test_utils/tests/graph_comparator_tests.cpp diff --git a/src/tests/ie_test_utils/common_test_utils/CMakeLists.txt b/src/tests/ie_test_utils/common_test_utils/CMakeLists.txt index c4a7e2dfe92..7c3e437cb13 100644 --- a/src/tests/ie_test_utils/common_test_utils/CMakeLists.txt +++ b/src/tests/ie_test_utils/common_test_utils/CMakeLists.txt @@ -10,6 +10,7 @@ function(add_common_utils ADD_TARGET_NAME) ROOT ${CMAKE_CURRENT_SOURCE_DIR} EXCLUDED_SOURCE_PATHS ${CMAKE_CURRENT_SOURCE_DIR}/gtest + ${CMAKE_CURRENT_SOURCE_DIR}/tests ADD_CPPLINT DEVELOPER_PACKAGE tests @@ -67,3 +68,5 @@ set(TARGET_NAME commonTestUtils) add_common_utils(${TARGET_NAME}) add_common_utils(${TARGET_NAME}_s USE_STATIC_IE) + +add_subdirectory(tests) diff --git a/src/tests/ie_test_utils/common_test_utils/tests/CMakeLists.txt b/src/tests/ie_test_utils/common_test_utils/tests/CMakeLists.txt new file mode 100644 index 00000000000..95ed4ff033a --- /dev/null +++ b/src/tests/ie_test_utils/common_test_utils/tests/CMakeLists.txt @@ -0,0 +1,11 @@ +set(TARGET_NAME commonUtilsTests) + +addIeTargetTest( + NAME ${TARGET_NAME} + ROOT ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDENCIES + commonTestUtils + LINK_LIBRARIES + commonTestUtils + ADD_CPPLINT +) \ No newline at end of file diff --git a/src/tests/ie_test_utils/common_test_utils/tests/graph_comparator_tests.cpp b/src/tests/ie_test_utils/common_test_utils/tests/graph_comparator_tests.cpp new file mode 100644 index 00000000000..1d3536cd5bd --- /dev/null +++ b/src/tests/ie_test_utils/common_test_utils/tests/graph_comparator_tests.cpp @@ -0,0 +1,622 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include + + +TEST(GraphComparatorTests, AllEnablePositiveCheck) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{1}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {1}, {0}); + auto add = std::make_shared(input, constant); + function_ref = std::make_shared(ngraph::NodeVector{ add}, ngraph::ParameterVector{ input }); + function = ov::clone_model(*function_ref); + } + comparator.enable(FunctionsComparator::NAMES) + .enable(FunctionsComparator::NODES) + .enable(FunctionsComparator::CONST_VALUES) + .enable(FunctionsComparator::PRECISIONS) + .enable(FunctionsComparator::ATTRIBUTES) + .enable(FunctionsComparator::RUNTIME_KEYS) + .enable(FunctionsComparator::TENSOR_NAMES); + + auto res = comparator.compare(function, function_ref); + ASSERT_TRUE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckbyDefault) { + FunctionsComparator comparator(FunctionsComparator::with_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto input2 = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto add = std::make_shared(input, input2); + function_ref = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input, input2 }); + } + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {12}); + auto add = std::make_shared(input, constant); + function = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + auto res = comparator.compare(function, function_ref); + ASSERT_FALSE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckResultsNumber) { + FunctionsComparator comparator(FunctionsComparator::with_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto input2 = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto add = std::make_shared(input, input2); + function_ref = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input, input2 }); + } + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {12}); + auto add = std::make_shared(input, constant); + auto result1 = std::make_shared(constant); + auto result2 = std::make_shared(add); + function = std::make_shared(ngraph::ResultVector{ result1, result2 }, ngraph::ParameterVector{ input }); + } + auto res = comparator.compare(function, function_ref); + ASSERT_FALSE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, NamesCheckPositive) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{1}); + input->set_friendly_name("new_name1"); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {1}, {0}); + constant->set_friendly_name("new_name2"); + auto add = std::make_shared(input, constant); + add->set_friendly_name("new_name3"); + function_ref = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{1}); + input->set_friendly_name("new_name1"); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {1}, {0}); + constant->set_friendly_name("new_name2"); + auto add = std::make_shared(input, constant); + add->set_friendly_name("new_name3"); + function = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::NAMES) + .enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_TRUE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, NamesCheckNegative) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{1}); + input->set_friendly_name("new_name1"); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {1}, {0}); + constant->set_friendly_name("new_name2"); + auto add = std::make_shared(input, constant); + add->set_friendly_name("new_name3"); + function_ref = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{1}); + input->set_friendly_name("new_name1"); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {1}, {0}); + constant->set_friendly_name("new_name2"); + auto add = std::make_shared(input, constant); + add->set_friendly_name("new_name3_different"); + function = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::NAMES) + .enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_FALSE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, ConstCheckWithoutEnable) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {0}); + auto add = std::make_shared(input, constant); + function_ref = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {12}); + auto add = std::make_shared(input, constant); + function = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_TRUE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, ConstCheckNegative) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {0}); + auto add = std::make_shared(input, constant); + function_ref = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {12}); + auto add = std::make_shared(input, constant); + function = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::CONST_VALUES) + .enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_FALSE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, TensorNamesCheckNegative) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{1}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {1}, {0}); + auto add = std::make_shared(input, constant); + function_ref = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + function = ov::clone_model(*function_ref); + add->get_input_tensor(0).set_names({"new_name"}); + } + comparator.enable(FunctionsComparator::TENSOR_NAMES) + .enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_FALSE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, TensorNamesCheckWithoutEnable) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{1}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {1}, {0}); + auto add = std::make_shared(input, constant); + function_ref = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + function = ov::clone_model(*function_ref); + add->get_input_tensor(0).set_names({"new_name"}); + } + comparator.enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_TRUE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckAttributesNegative) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ov::element::f32, ov::Shape{ 1, 3, 12, 12 }); + auto const_weights = ov::opset8::Constant::create(ov::element::f16, + ov::Shape{ 1, 3, 3, 3 }, + { 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + auto convert_ins1 = std::make_shared(const_weights, ov::element::f32); + auto conv = std::make_shared(input, + convert_ins1, + ov::Strides{ 1, 1 }, + ov::CoordinateDiff{ 1, 1 }, + ov::CoordinateDiff{ 1, 1 }, + ov::Strides{ 1, 1 }); + function_ref = std::make_shared(ngraph::NodeVector{ conv }, ngraph::ParameterVector{ input }); + } + { + auto input = std::make_shared(ov::element::f32, ov::Shape{ 1, 3, 12, 12 }); + auto const_weights = ov::opset8::Constant::create(ov::element::f16, + ov::Shape{ 1, 3, 3, 3 }, + { 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + auto convert_ins1 = std::make_shared(const_weights, ov::element::f32); + auto conv = std::make_shared(input, + convert_ins1, + ov::Strides{ 1, 1 }, + ov::CoordinateDiff{ 0, 0 }, + ov::CoordinateDiff{ 0, 0 }, + ov::Strides{ 1, 1 }); + function = std::make_shared(ngraph::NodeVector{ conv }, ngraph::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::ATTRIBUTES) + .enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_FALSE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckPrecisionsNegative) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {0}); + auto add = std::make_shared(input, constant); + function_ref = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + { + auto input = std::make_shared(ngraph::element::f32, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::f32, {3}, {0}); + auto add = std::make_shared(input, constant); + function = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::PRECISIONS) + .enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_FALSE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckPrecisionsWithoutEnable) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {0}); + auto add = std::make_shared(input, constant); + function_ref = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + { + auto input = std::make_shared(ngraph::element::f32, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::f32, {3}, {0}); + auto add = std::make_shared(input, constant); + function = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_TRUE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckRTInfo) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {0}); + auto add = std::make_shared(input, constant); + add->get_rt_info()["my_info"] = 42; + function_ref = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {0}); + auto add = std::make_shared(input, constant); + function = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::RUNTIME_KEYS) + .enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_FALSE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckRTInfoReverse) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {0}); + auto add = std::make_shared(input, constant); + function_ref = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {0}); + auto add = std::make_shared(input, constant); + add->get_rt_info()["my_info"] = 42; + function = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::RUNTIME_KEYS) + .enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_TRUE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckRTInfoInput) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {0}); + auto add = std::make_shared(input, constant); + add->input(0).get_rt_info()["my_info"] = 42; + function_ref = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {0}); + auto add = std::make_shared(input, constant); + function = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::RUNTIME_KEYS) + .enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_FALSE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckRTInfoOutput) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {0}); + auto add = std::make_shared(input, constant); + add->output(0).get_rt_info()["my_info"] = 42; + function_ref = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{3}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {3}, {0}); + auto add = std::make_shared(input, constant); + function = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::RUNTIME_KEYS) + .enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_FALSE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckTensorIteratorPositive) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto X = std::make_shared(ngraph::element::f32, ngraph::Shape{2, 1, 16}); + auto Y = std::make_shared(ngraph::element::f32, ngraph::Shape{1, 128}); + + auto Xi = std::make_shared(ngraph::element::f32, ngraph::Shape{1, 1, 16}); + auto Yi = std::make_shared(ngraph::element::f32, ngraph::Shape{1, 128}); + + // Body + auto axis = ov::opset8::Constant::create(ngraph::element::i64, ngraph::Shape{}, {0}); + auto squeeze = std::make_shared(Xi, axis); + + auto w_val = std::vector(384*16, 0); + auto r_val = std::vector(384*128, 0); + auto b_val = std::vector(384, 0); + auto W = ov::opset8::Constant::create(ngraph::element::f32, ngraph::Shape{384, 16}, w_val); + auto R = ov::opset8::Constant::create(ngraph::element::f32, ngraph::Shape{384, 128}, r_val); + auto B = ov::opset8::Constant::create(ngraph::element::f32, ngraph::Shape{384}, b_val); + + auto gru_cell = std::make_shared(squeeze, Yi, W, R, B, 128); + auto res_1 = std::make_shared(gru_cell); + auto unsqueeze = std::make_shared(gru_cell, axis); + auto res_2 = std::make_shared(unsqueeze); + auto body = std::make_shared(ngraph::OutputVector{res_1, res_2}, + ngraph::ParameterVector{Xi, Yi}); + + auto tensor_iterator = std::make_shared(); + tensor_iterator->set_body(body); + + tensor_iterator->set_sliced_input(Xi, X, 0, 1, 1, -1, 0); + tensor_iterator->set_merged_input(Yi, Y, res_1); + + auto out0 = tensor_iterator->get_iter_value(res_1, -1); + auto out1 = tensor_iterator->get_concatenated_slices(res_2, 0, 1, 1, -1, 0); + + auto res_ti_1 = std::make_shared(tensor_iterator->output(1)); + function_ref = std::make_shared(ngraph::NodeVector{res_ti_1}, + ngraph::ParameterVector{X, Y}); + function = ov::clone_model(*function_ref); + } + comparator.enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_TRUE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckLoopPositive) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto X = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto Y = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto M = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + + // Set up the cell body, a function from (Xi, Yi) -> (Zo) + // Body parameters + auto Xi = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto Yi = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto M_body = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto body_condition = std::make_shared(ov::element::boolean, ov::Shape{1}, true); + + auto trip_count = std::make_shared(ngraph::element::i64, ov::Shape{1}, 3); + auto exec_condition = std::make_shared(ngraph::element::boolean, ov::Shape{1}, true); + // Body + auto sum = std::make_shared(Xi, Yi); + auto Zo = std::make_shared(sum, M_body); + auto body = std::make_shared(ov::OutputVector{body_condition, Zo}, + ov::ParameterVector{Xi, Yi, M_body}); + + auto loop = std::make_shared(trip_count, exec_condition); + loop->set_function(body); + + loop->set_invariant_input(Xi, X); + loop->set_invariant_input(Yi, Y); + loop->set_merged_input(M_body, M, Zo); + + loop->set_special_body_ports(ov::opset8::Loop::SpecialBodyPorts{-1, 0}); + + // Output is last Zo + auto result = std::make_shared(loop->get_iter_value(Zo, -1)); + function_ref = std::make_shared(ov::ResultVector{result}, ov::ParameterVector{X, Y, M}); + function = ov::clone_model(*function_ref); + } + comparator.enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_TRUE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckSinksPositive) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto arg = std::make_shared(ov::element::f32, ov::Shape{1, 1}); + auto init_const = ov::opset8::Constant::create(ov::element::f32, ov::Shape{1, 1}, {0}); + const std::string variable_name("variable0"); + auto variable = std::make_shared(ngraph::VariableInfo{ov::PartialShape::dynamic(), + ov::element::dynamic, variable_name}); + + auto read = std::make_shared(init_const, variable); + auto read2 = std::make_shared(init_const, variable); + auto add = std::make_shared(arg, read); + auto add2 = std::make_shared(arg, read2); + auto assign = std::make_shared(add, variable); + auto assign2 = std::make_shared(add, variable); + + auto res = std::make_shared(add); + auto res2 = std::make_shared(add2); + + function_ref = std::make_shared(ov::ResultVector({res, res2}), ov::SinkVector({assign, assign2}), + ov::ParameterVector({arg})); + function = ov::clone_model(*function_ref); + } + comparator.enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_TRUE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckSinksNegative) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto arg = std::make_shared(ov::element::f32, ov::Shape{1, 1}); + auto init_const = ov::opset8::Constant::create(ov::element::f32, ov::Shape{1, 1}, {0}); + const std::string variable_name("variable0"); + auto variable = std::make_shared(ngraph::VariableInfo{ov::PartialShape::dynamic(), + ov::element::dynamic, variable_name}); + + auto read = std::make_shared(init_const, variable); + auto read2 = std::make_shared(init_const, variable); + auto add = std::make_shared(arg, read); + auto add2 = std::make_shared(arg, read2); + auto assign = std::make_shared(add, variable); + auto assign2 = std::make_shared(add, variable); + + auto res = std::make_shared(add); + auto res2 = std::make_shared(add2); + + function_ref = std::make_shared(ov::ResultVector({res, res2}), ov::SinkVector({assign, assign2}), + ov::ParameterVector({arg})); + } + + { + auto arg = std::make_shared(ov::element::f32, ov::Shape{1, 1}); + auto init_const = ov::opset8::Constant::create(ov::element::f32, ov::Shape{1, 1}, {0}); + const std::string variable_name("variable_different"); + auto variable = std::make_shared(ngraph::VariableInfo{ov::PartialShape::dynamic(), + ov::element::dynamic, variable_name}); + + auto read = std::make_shared(init_const, variable); + auto read2 = std::make_shared(init_const, variable); + auto add = std::make_shared(arg, read); + auto add2 = std::make_shared(arg, read2); + auto assign = std::make_shared(add, variable); + auto assign2 = std::make_shared(add, variable); + + auto res = std::make_shared(add); + auto res2 = std::make_shared(add2); + + function = std::make_shared(ov::ResultVector({res, res2}), ov::SinkVector({assign, assign2}), + ov::ParameterVector({arg})); + } + comparator.enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_FALSE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, DisableCheck) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{1}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {1}, {0}); + auto add = std::make_shared(input, constant); + function_ref = std::make_shared(ngraph::NodeVector{ add }, ngraph::ParameterVector{ input }); + function = ov::clone_model(*function_ref); + } + comparator.enable(FunctionsComparator::NODES); + comparator.disable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_FALSE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckAccuracyPositive) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{1}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {1}, {0}); + auto add = std::make_shared(input, constant); + function_ref = std::make_shared(ngraph::NodeVector{ add}, ngraph::ParameterVector{ input }); + } + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{1}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {1}, {0}); + auto add = std::make_shared(input, constant); + function = std::make_shared(ngraph::NodeVector{ add}, ngraph::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::ACCURACY); + auto res = comparator.compare(function, function_ref); + ASSERT_TRUE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckAccuracyNegative) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{1}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {1}, {12}); + auto add = std::make_shared(input, constant); + function_ref = std::make_shared(ngraph::NodeVector{ add}, ngraph::ParameterVector{ input }); + } + { + auto input = std::make_shared(ngraph::element::i64, ov::Shape{1}); + auto constant = ov::opset8::Constant::create(ngraph::element::i64, {1}, {200}); + auto add = std::make_shared(input, constant); + function = std::make_shared(ngraph::NodeVector{ add}, ngraph::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::ACCURACY); + auto res = comparator.compare(function, function_ref); + ASSERT_FALSE(res.valid) << res.message; +} + +TEST(GraphComparatorTests, CheckAccuracyNotEnabled) { + FunctionsComparator comparator(FunctionsComparator::no_default()); + std::shared_ptr function, function_ref; + { + auto input = std::make_shared(ov::element::f32, ov::Shape{ 1, 3, 12, 12 }); + auto const_weights = ov::opset8::Constant::create(ov::element::f16, + ov::Shape{ 1, 3, 3, 3 }, + { 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + auto convert_ins1 = std::make_shared(const_weights, ov::element::f32); + auto conv = std::make_shared(input, + convert_ins1, + ov::Strides{ 1, 1 }, + ov::CoordinateDiff{ 1, 1 }, + ov::CoordinateDiff{ 1, 1 }, + ov::Strides{ 1, 1 }); + function_ref = std::make_shared(ngraph::NodeVector{ conv }, ngraph::ParameterVector{ input }); + } + { + auto input = std::make_shared(ov::element::f32, ov::Shape{ 1, 3, 12, 12 }); + auto const_weights = ov::opset8::Constant::create(ov::element::f16, + ov::Shape{ 1, 3, 3, 3 }, + { 1, 9, 3, 4, 5, 6, 7, 8, 9, 1, 12, 3, 9, 5, 0, 7, 8, 9, 1, 2, 12, 4, 9, 6, 7, 8, 9 }); + auto convert_ins1 = std::make_shared(const_weights, ov::element::f32); + auto conv = std::make_shared(input, + convert_ins1, + ov::Strides{ 1, 1 }, + ov::CoordinateDiff{ 1, 1 }, + ov::CoordinateDiff{ 1, 1 }, + ov::Strides{ 1, 1 }); + function = std::make_shared(ngraph::NodeVector{ conv }, ngraph::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::NODES); + auto res = comparator.compare(function, function_ref); + ASSERT_TRUE(res.valid) << res.message; +} From 16a5962698a31ad44c0a2f9ba6dc9faa8eb6ed1b Mon Sep 17 00:00:00 2001 From: Pavel Esir Date: Thu, 31 Mar 2022 14:07:07 +0300 Subject: [PATCH 29/39] [MO] pad fusing fix (#10453) * pad fusing fix * added unit-tests for pad fusing fix * fixed port reconnecting * Update tools/mo/openvino/tools/mo/middle/passes/fusing/mark_unfused_nodes.py Co-authored-by: Evgenya Stepyreva Co-authored-by: Evgenya Stepyreva --- tools/mo/openvino/tools/mo/middle/fusings.py | 4 +- .../openvino/tools/mo/middle/passes/conv.py | 13 +- .../passes/fusing/mark_unfused_nodes.py | 13 ++ tools/mo/unit_tests/mo/front/conv_test.py | 198 ++++++++++++++++++ 4 files changed, 222 insertions(+), 6 deletions(-) create mode 100644 tools/mo/unit_tests/mo/front/conv_test.py diff --git a/tools/mo/openvino/tools/mo/middle/fusings.py b/tools/mo/openvino/tools/mo/middle/fusings.py index c313e6bfc2e..924703012a0 100644 --- a/tools/mo/openvino/tools/mo/middle/fusings.py +++ b/tools/mo/openvino/tools/mo/middle/fusings.py @@ -16,7 +16,7 @@ from openvino.tools.mo.middle.passes.fusing.decomposition import convert_scale_s from openvino.tools.mo.middle.passes.fusing.fuse_grouped_conv import grouped_convolutions_fusing from openvino.tools.mo.middle.passes.fusing.fuse_linear_ops import fuse_linear_ops from openvino.tools.mo.middle.passes.fusing.fuse_linear_seq import fuse_mul_add_sequence -from openvino.tools.mo.middle.passes.fusing.mark_unfused_nodes import mark_unfused_nodes +from openvino.tools.mo.middle.passes.fusing.mark_unfused_nodes import mark_unfused_nodes, mark_shape_of_sugraph_as_unfusable from openvino.tools.mo.middle.passes.fusing.resnet_optimization import stride_optimization from openvino.tools.mo.middle.pattern_match import for_graph_and_each_sub_graph_recursively from openvino.tools.mo.middle.replacement import MiddleReplacementPattern @@ -42,6 +42,7 @@ class Fusing(MiddleReplacementPattern): argv = graph.graph['cmd_params'] layout = graph.graph['layout'] + mark_shape_of_sugraph_as_unfusable(graph) for_graph_and_each_sub_graph_recursively(graph, fuse_pad) for_graph_and_each_sub_graph_recursively(graph, lambda G: G.clean_up()) @@ -93,6 +94,7 @@ class Fusing(MiddleReplacementPattern): MulFakeQuantizeFuse().find_and_replace_pattern(graph) for_graph_and_each_sub_graph_recursively(graph, lambda G: G.clean_up()) + mark_shape_of_sugraph_as_unfusable(graph) for_graph_and_each_sub_graph_recursively(graph, fuse_pad) for_graph_and_each_sub_graph_recursively(graph, lambda G: G.clean_up()) diff --git a/tools/mo/openvino/tools/mo/middle/passes/conv.py b/tools/mo/openvino/tools/mo/middle/passes/conv.py index 392d78b7908..2408dc8fc87 100644 --- a/tools/mo/openvino/tools/mo/middle/passes/conv.py +++ b/tools/mo/openvino/tools/mo/middle/passes/conv.py @@ -16,8 +16,11 @@ from openvino.tools.mo.middle.pattern_match import apply_pattern def pad_op_transform(graph: Graph, match: dict): op = match['op'] - pad_op = match['pad_op'] - input_data = pad_op.in_node(0) + pad_op: Node = match['pad_op'] + + # to keep reshape-ability if Pad receives pads_begin/pads_end from shape subgraph + if pad_op.in_port(1).get_source().node.soft_get('can_be_fused') is False: + return if pad_op.mode != 'constant': log.info('The pad node "{}" with pad mode "{}" cannot be fused.'.format(pad_op.soft_get('name'), pad_op.mode)) @@ -48,9 +51,9 @@ def pad_op_transform(graph: Graph, match: dict): if op.type == 'Pooling': op['exclude_pad'] = False assert (graph[match['pad_output'].node][match['op'].node][0]['in'] == 0) - edge_attrs = graph.get_edge_data(match['pad_output'].id, match['op'].id)[0] - graph.remove_edge(match['pad_output'].id, match['op'].id) - graph.add_edge(input_data.id, match['op'].id, **{'in': 0, **edge_attrs}) + + match['op'].in_port(0).disconnect() + pad_op.in_port(0).get_connection().add_destination(match['op'].in_port(0)) def fuse_pad(graph: Graph): diff --git a/tools/mo/openvino/tools/mo/middle/passes/fusing/mark_unfused_nodes.py b/tools/mo/openvino/tools/mo/middle/passes/fusing/mark_unfused_nodes.py index 9bf1a252313..4e041a6d8c1 100644 --- a/tools/mo/openvino/tools/mo/middle/passes/fusing/mark_unfused_nodes.py +++ b/tools/mo/openvino/tools/mo/middle/passes/fusing/mark_unfused_nodes.py @@ -5,6 +5,7 @@ import logging as log import re from openvino.tools.mo.graph.graph import Node, Graph +from openvino.tools.mo.middle.MarkSubgraphsWithCorrectLayout import MarkSubGraphsWithCorrectLayout from openvino.tools.mo.middle.passes.fusing.helpers import get_value_id @@ -39,3 +40,15 @@ def mark_unfused_nodes(graph: Graph, regex_masks: str): _check_lin_op(node, graph.graph['layout']) +def mark_shape_of_sugraph_as_unfusable(graph: Graph): + def condition_to_continue(node: Node): + for port in node.out_ports().values(): + if port.data.get_value() is None: + return False + return True + + starting_nodes = graph.get_op_nodes(op='ShapeOf') + shapeof_subgraph_nodes = MarkSubGraphsWithCorrectLayout.bfs(starting_nodes, set(), condition_to_continue) + + for node in shapeof_subgraph_nodes: + node['can_be_fused'] = False diff --git a/tools/mo/unit_tests/mo/front/conv_test.py b/tools/mo/unit_tests/mo/front/conv_test.py new file mode 100644 index 00000000000..0612013ab92 --- /dev/null +++ b/tools/mo/unit_tests/mo/front/conv_test.py @@ -0,0 +1,198 @@ +# Copyright (C) 2018-2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import unittest + +import numpy as np + +from openvino.tools.mo.front.common.partial_infer.concat import concat_infer +from openvino.tools.mo.front.common.partial_infer.utils import shape_array +from openvino.tools.mo.middle.passes.conv import fuse_pad +from openvino.tools.mo.middle.passes.fusing.mark_unfused_nodes import mark_shape_of_sugraph_as_unfusable +from openvino.tools.mo.middle.passes.infer import partial_infer +from openvino.tools.mo.middle.pattern_match import for_graph_and_each_sub_graph_recursively +from openvino.tools.mo.ops.convolution import Convolution +from openvino.tools.mo.ops.elementwise import eltwise_infer +from openvino.tools.mo.ops.gather import Gather +from openvino.tools.mo.ops.pad import Pad +from openvino.tools.mo.ops.shape import Shape +from openvino.tools.mo.utils.ir_engine.compare_graphs import compare_graphs +from unit_tests.utils.graph import build_graph, regular_op_with_empty_data, result, connect, \ + shaped_parameter, valued_const_with_data + + +class PadFusingTest(unittest.TestCase): + + # standard case: not in the shape subgraph + def test_pad_fusing(self): + nodes = { + **shaped_parameter('input', shape_array([1, 3, 248, 248])), + + **valued_const_with_data('pads_begin', shape_array([0, 0, 1, 1])), + **valued_const_with_data('pads_end', shape_array([0, 0, 1, 1])), + **valued_const_with_data('fill_value', shape_array(0.0)), + **valued_const_with_data('weights', shape_array(np.zeros([3, 16, 4, 4]))), + + **regular_op_with_empty_data('pad', {'type': 'Pad', + 'op': 'Pad', + 'infer': Pad.infer, + 'mode': 'constant'}), + + **regular_op_with_empty_data('conv', {'type': 'Convolution', + 'op': 'Convolution', + 'infer': Convolution.infer, + # zeros, no paddings + 'pad': np.array([[0, 0], [0, 0], [0, 0], [0, 0]]), + 'dilation': np.array([1, 1, 1, 1]), + 'stride': np.array([1, 1, 1, 1]), + 'group': 1, + 'kernel_spatial_idx': np.array([2, 3]), + 'output': 64, + 'spatial_dims': np.array([2, 3]), + 'channel_dims': np.array([1]), + 'batch_dims': np.array([0]), + 'input_feature_channel': 1, + 'output_feature_channel': 0}), + **result(), + } + + graph = build_graph(nodes_attrs=nodes, edges=[ + *connect('input', '0:pad'), + *connect('pads_begin', '1:pad'), + *connect('pads_end', '2:pad'), + *connect('fill_value', '3:pad'), + *connect('pad', '0:conv'), + *connect('weights', '1:conv'), + *connect('conv', 'output'), + ], nodes_with_edges_only=True) + + graph.graph['layout'] = 'NCHW' + graph.stage = 'middle' + + graph = partial_infer(graph) + mark_shape_of_sugraph_as_unfusable(graph) + for_graph_and_each_sub_graph_recursively(graph, fuse_pad) + graph.clean_up() + + conv_fused_with_pad = regular_op_with_empty_data('conv', {'type': 'Convolution', + 'op': 'Convolution', + # ones are taken from fused Pad + 'pad': np.array([[0, 0], [0, 0], [1, 1], [1, 1]]), + 'dilation': np.array([1, 1, 1, 1]), + 'stride': np.array([1, 1, 1, 1]), + 'group': 1, + 'kernel_spatial_idx': np.array([2, 3]), + 'output': 64, + 'spatial_dims': np.array([2, 3]), + 'channel_dims': np.array([1]), + 'batch_dims': np.array([0]), + 'input_feature_channel': 1, + 'output_feature_channel': 0, + + 'infer': Convolution.infer}) + + graph_ref = build_graph(nodes_attrs=nodes, update_attributes=conv_fused_with_pad, edges=[ + *connect('input', '0:conv'), + *connect('weights', '1:conv'), + *connect('conv', 'output'), + ], nodes_with_edges_only=True) + graph_ref.graph['layout'] = 'NCHW' + graph_ref.stage = 'middle' + + (flag, resp) = compare_graphs(graph, graph_ref, 'output', check_op_attrs=True) + self.assertTrue(flag, resp) + + # Pad in the shape subgraph + def test_pad_fusing_shape_subgraph(self): + nodes = { + **shaped_parameter('input', shape_array([1, 3, 1020, 1020])), + **regular_op_with_empty_data('input_shape', {'type': 'ShapeOf', 'op': 'ShapeOf', 'output_type': np.int64, + 'infer': Shape.infer}), + **regular_op_with_empty_data('gathered_shape', {'type': 'Gather', 'batch_dims': 0, 'infer': Gather.infer}), + **valued_const_with_data('axis', np.array([0])), + **valued_const_with_data('indices', np.array([2, 3])), + + **regular_op_with_empty_data('div', {'type': 'Div', + 'infer': lambda node: eltwise_infer(node, lambda a, b: a / b)}), + **regular_op_with_empty_data('sub_1', {'type': 'Sub', + 'infer': lambda node: eltwise_infer(node, lambda a, b: a - b)}), + **regular_op_with_empty_data('sub_2', {'type': 'Sub', + 'infer': lambda node: eltwise_infer(node, lambda a, b: a - b)}), + + **valued_const_with_data('div_const', shape_array([2])), + **valued_const_with_data('sub_const', shape_array([512])), + + **regular_op_with_empty_data('pad', {'type': 'Pad', + 'op': 'Pad', + 'infer': Pad.infer, + 'mode': 'constant'}), + + **regular_op_with_empty_data('concat', {'type': 'Concat', + 'op': 'Concat', + 'axis': 0, + 'infer': concat_infer}), + + **valued_const_with_data('pad_end', shape_array([0, 0, 0, 0])), + **valued_const_with_data('blank_zeros', shape_array([0, 0])), + + **regular_op_with_empty_data('conv', {'type': 'Convolution', + 'op': 'Convolution', + + 'pad': np.array([[0, 0], [0, 0], [0, 0], [0, 0]]), + 'dilation': np.array([1, 1, 1, 1]), + 'stride': np.array([1, 1, 1, 1]), + 'group': 1, + 'kernel_spatial_idx': np.array([2, 3]), + 'output': 64, + 'spatial_dims': np.array([2, 3]), + 'channel_dims': np.array([1]), + 'batch_dims': np.array([0]), + 'input_feature_channel': 1, + 'output_feature_channel': 0, + + 'infer': Convolution.infer}), + **valued_const_with_data('weights', shape_array(np.zeros([3, 16, 4, 4]))), + **result(), + } + + graph = build_graph(nodes_attrs=nodes, + update_attributes={ + 'gathered_shape_d': {'kind': 'data', 'value': shape_array([256, 256]), + 'shape': shape_array([2])}}, + edges=[ + *connect('input', 'input_shape', skip_data=True), + *connect('input_shape', '0:gathered_shape'), + *connect('indices', '1:gathered_shape'), + *connect('axis', '2:gathered_shape'), + + *connect('gathered_shape', 'sub_1'), + *connect('sub_const', 'sub_1'), + *connect('sub_1', 'div'), + *connect('div_const', 'div'), + *connect('div', '0:sub_2'), + *connect('sub_1', '1:sub_2'), + *connect('input', '0:pad'), + + *connect('blank_zeros', '0:concat'), + *connect('sub_2', '1:concat'), + *connect('concat', '1:pad'), + + *connect('pad_end', '2:pad'), + *connect('pad', '0:conv'), + *connect('weights', '1:conv'), + *connect('conv', 'output'), + ], nodes_with_edges_only=True) + + graph.graph['layout'] = 'NCHW' + graph.stage = 'middle' + + graph = partial_infer(graph) + + # graph must remain unchanged + graph_ref = graph.copy() + + mark_shape_of_sugraph_as_unfusable(graph) + for_graph_and_each_sub_graph_recursively(graph, fuse_pad) + + (flag, resp) = compare_graphs(graph, graph_ref, 'output', check_op_attrs=True) + self.assertTrue(flag, resp) From a87e8f7880c9aea054f5bb4f7c2371de92242731 Mon Sep 17 00:00:00 2001 From: Smirnov Grigorii Date: Thu, 31 Mar 2022 14:07:41 +0300 Subject: [PATCH 30/39] moved TransformationsTestsF method's definitions from .hpp to .cpp (#11359) * moved * fix style --- .../common_test_utils/ngraph_test_utils.cpp | 46 +++++++++++++++++ .../common_test_utils/ngraph_test_utils.hpp | 49 +++---------------- 2 files changed, 52 insertions(+), 43 deletions(-) diff --git a/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.cpp b/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.cpp index b2b2fea8f08..b25dfbe771e 100644 --- a/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.cpp +++ b/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.cpp @@ -4,6 +4,52 @@ #include "ngraph_test_utils.hpp" +TransformationTestsF::TransformationTestsF() : comparator(FunctionsComparator::no_default()) { + m_unh = std::make_shared(); + comparator.enable(FunctionsComparator::CmpValues::NODES); + comparator.enable(FunctionsComparator::CmpValues::PRECISIONS); + comparator.enable(FunctionsComparator::CmpValues::RUNTIME_KEYS); + // TODO: enable attributes and constant values comparison by default XXX-68694 + // comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + // comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); +} + +void TransformationTestsF::SetUp() { + manager.register_pass(m_unh); + manager.register_pass(); +} + +void TransformationTestsF::TearDown() { + auto cloned_function = ngraph::clone_function(*function); + if (!function_ref) { + function_ref = cloned_function; + } + + manager.register_pass(m_unh, m_soft_names_comparison); + manager.run_passes(function); + if (!m_disable_rt_info_check) { + ASSERT_NO_THROW(check_rt_info(function)); + } + + if (comparator.should_compare(FunctionsComparator::ACCURACY)) { + auto acc_comparator = FunctionsComparator::no_default(); + acc_comparator.enable(FunctionsComparator::CmpValues::ACCURACY); + auto res = acc_comparator.compare(function, cloned_function); + ASSERT_TRUE(res.valid) << res.message; + comparator.disable(FunctionsComparator::CmpValues::ACCURACY); + } + auto res = comparator.compare(function, function_ref); + ASSERT_TRUE(res.valid) << res.message; +} + +void TransformationTestsF::disable_rt_info_check() { + m_disable_rt_info_check = true; +} + +void TransformationTestsF::enable_soft_names_comparison() { + m_soft_names_comparison = true; +} + void init_unique_names(std::shared_ptr f, const std::shared_ptr& unh) { ngraph::pass::Manager manager; manager.register_pass(unh); diff --git a/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.hpp b/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.hpp index c79898e1930..126c256a846 100644 --- a/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.hpp +++ b/src/tests/ie_test_utils/common_test_utils/ngraph_test_utils.hpp @@ -30,54 +30,17 @@ using TransformationTests = CommonTestUtils::TestsCommon; class TransformationTestsF : public CommonTestUtils::TestsCommon { public: - TransformationTestsF() : comparator(FunctionsComparator::no_default()) { - m_unh = std::make_shared(); - comparator.enable(FunctionsComparator::CmpValues::NODES); - comparator.enable(FunctionsComparator::CmpValues::PRECISIONS); - comparator.enable(FunctionsComparator::CmpValues::RUNTIME_KEYS); - // TODO: enable attributes and constant values comparison by default XXX-68694 - // comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); - // comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); - } + TransformationTestsF(); - void SetUp() override { - manager.register_pass(m_unh); - manager.register_pass(); - } + void SetUp() override; - void TearDown() override { - auto cloned_function = ngraph::clone_function(*function); - if (!function_ref) { - function_ref = cloned_function; - } - - manager.register_pass(m_unh, m_soft_names_comparison); - manager.run_passes(function); - if (!m_disable_rt_info_check) { - ASSERT_NO_THROW(check_rt_info(function)); - } - - if (comparator.should_compare(FunctionsComparator::ACCURACY)) { - auto acc_comparator = FunctionsComparator::no_default(); - acc_comparator.enable(FunctionsComparator::CmpValues::ACCURACY); - auto res = acc_comparator.compare(function, cloned_function); - ASSERT_TRUE(res.valid) << res.message; - comparator.disable(FunctionsComparator::CmpValues::ACCURACY); - } - - auto res = comparator.compare(function, function_ref); - ASSERT_TRUE(res.valid) << res.message; - } + void TearDown() override; // TODO: this is temporary solution to disable rt info checks that must be applied by default // first tests must be fixed then this method must be removed XXX-68696 - void disable_rt_info_check() { - m_disable_rt_info_check = true; - } + void disable_rt_info_check(); - void enable_soft_names_comparison() { - m_soft_names_comparison = true; - } + void enable_soft_names_comparison(); std::shared_ptr function, function_ref; ngraph::pass::Manager manager; @@ -103,4 +66,4 @@ size_t count_ops_of_type(const std::shared_ptr& f) { } return count; -} \ No newline at end of file +} From 9185f03e77253f66569e7c76d1addea7465a7b85 Mon Sep 17 00:00:00 2001 From: Alexandra Sidorova Date: Thu, 31 Mar 2022 14:46:55 +0300 Subject: [PATCH 31/39] Added specification for Eye-9 (#11104) * Added specification for EyeLike-9 * Update docs/ops/generation/EyeLike_9.md * removed batch from TF * minor fix * Applied comment by Anton * Added new example with dynamic output, added corner case * Fixed corner case description * Rename matrix * applied comments by Yuan * Added diag_idx as input, minor fixes, renaming * added support of batch_shape from TF Co-authored-by: Andrei Kochin --- .../Operations_specifications.md | 1 + docs/ops/generation/Eye_9.md | 170 ++++++++++++++++++ docs/ops/opset9.md | 1 + 3 files changed, 172 insertions(+) create mode 100644 docs/ops/generation/Eye_9.md diff --git a/docs/OV_Runtime_UG/Operations_specifications.md b/docs/OV_Runtime_UG/Operations_specifications.md index e7366483166..99c911c1feb 100644 --- a/docs/OV_Runtime_UG/Operations_specifications.md +++ b/docs/OV_Runtime_UG/Operations_specifications.md @@ -60,6 +60,7 @@ openvino_docs_ops_detection_ExperimentalDetectronROIFeatureExtractor_6 openvino_docs_ops_sort_ExperimentalDetectronTopKROIs_6 openvino_docs_ops_movement_ExtractImagePatches_3 + openvino_docs_ops_generation_Eye_9 openvino_docs_ops_quantization_FakeQuantize_1 openvino_docs_ops_arithmetic_FloorMod_1 openvino_docs_ops_arithmetic_Floor_1 diff --git a/docs/ops/generation/Eye_9.md b/docs/ops/generation/Eye_9.md new file mode 100644 index 00000000000..28af3d9db7b --- /dev/null +++ b/docs/ops/generation/Eye_9.md @@ -0,0 +1,170 @@ +## Eye {#openvino_docs_ops_generation_Eye_9} + +**Versioned name**: *Eye-9* + +**Category**: *Generation* + +**Short description**: *Eye* operation generates shift matrix or a batch of matrices. + +**Detailed description**: + +*Eye* operation generates an identity matrix or a batch matrices with ones on the diagonal and zeros everywhere else. The index of the diagonal to be populated with ones is given by `diagonal_index`: `output[..., i, i + diagonal_index] = 1`. + + +Example 1. *Eye* output with `output_type` = `f32`: + +``` +num_rows = 3 + +output = [[1. 0. 0.] + [0. 1. 0.] + [0. 0. 1.]] +``` + +Example 2. *Eye* output with `output_type` = `i32`: + +``` +num_rows = 3 + +num_columns = 4 + +diagonal_index = 2 + +output = [[0 0 1 0] + [0 0 0 1] + [0 0 0 0]] +``` + +Example 3. *Eye* output with `output_type` = `i32`: + +``` +num_rows = 3 + +num_columns = 4 + +diagonal_index = -1 + +output = [[0 0 0 0] + [1 0 0 0] + [0 1 0 0]] +``` + +Example 4. *Eye* output with `output_type` = `f16`: + +``` +num_rows = 2 + +diagonal_index = 5 + +batch_shape = [1, 2] + +output = [[[[0. 0.] + [0. 0.]] + [[0. 0.] + [0. 0.]]]] +``` + +**Attributes**: + +* *output_type* + + * **Description**: the type of the output + * **Range of values**: any numeric type + * **Type**: string + * **Required**: *Yes* + + +**Inputs**: + +* **1**: `num_rows` - scalar or 1D tensor with 1 non-negative element of type *T_NUM* describing the number of rows in matrix. **Required.** + +* **2**: `num_columns` - scalar or 1D tensor with 1 non-negative element of type *T_NUM* describing the number of columns in matrix. This input is optional, and its default value equals to `num_rows`. + +* **3**: `diagonal_index` - scalar or 1D tensor with element of type *T_NUM* describing the index of the diagonal to be populated. A positive value refers to an upper diagonal and a negative value refers to a lower diagonal. Value `0` populates the main diagonal. If `diagonal_index` is a positive value and is not smaller than `num_rows` or if `diagonal_index` is a negative value and is not larger than `num_columns`, the matrix will be filled with only zeros. This input is optional, and its default value equals to `0`. + +* **4**: `batch_shape` - 1D tensor with non-negative values of type *T_NUM* defines leading batch dimensions of output shape. If `batch_shape` is an empty list, *Eye* operation generates a 2D tensor (matrix). This input is optional, and its default value equal to an empty tensor. + + +**Outputs**: + +* **1**: A tensor with the type specified by the *output_type* attribute. The shape is `batch_shape + [num_rows, num_columns]` + +**Types** + +* *T_NUM*: `int32` or `int64`. + +**Examples** + +*Example 1* + +```xml + + + + + + + + 10 + 10 + + + +``` + +*Example 2* + +```xml + + + + + + + + + 5 + -1 + + + +``` + +*Example 3* + +```xml + + + + + + + + + -1 + -1 + + + +``` + +*Example 4* + +```xml + + + + + + + + + + 2 + 3 + -1 + -1 + + + +``` diff --git a/docs/ops/opset9.md b/docs/ops/opset9.md index 803fd4647bc..b5454764c78 100644 --- a/docs/ops/opset9.md +++ b/docs/ops/opset9.md @@ -60,6 +60,7 @@ declared in `namespace opset9`. * [ExperimentalDetectronROIFeatureExtractor_6](detection/ExperimentalDetectronROIFeatureExtractor_6.md) * [ExperimentalDetectronTopKROIs_6](sort/ExperimentalDetectronTopKROIs_6.md) * [ExtractImagePatches](movement/ExtractImagePatches_3.md) +* [Eye](generation/Eye_9.md) * [FakeQuantize](quantization/FakeQuantize_1.md) * [Floor](arithmetic/Floor_1.md) * [FloorMod](arithmetic/FloorMod_1.md) From 1d247815be1bad8c1cbaed2d82c38b49d4269c6a Mon Sep 17 00:00:00 2001 From: Maxim Andronov Date: Thu, 31 Mar 2022 15:42:10 +0300 Subject: [PATCH 32/39] Don't execute reference::strided_slice if input/output tensor is empty (#11337) --- .../functional/op_reference/strided_slice.cpp | 14 ++++++++++++++ .../src/runtime/reference/strided_slice.cpp | 9 +++++++++ 2 files changed, 23 insertions(+) diff --git a/docs/template_plugin/tests/functional/op_reference/strided_slice.cpp b/docs/template_plugin/tests/functional/op_reference/strided_slice.cpp index c241b7d3b6e..5ce1bebf63b 100644 --- a/docs/template_plugin/tests/functional/op_reference/strided_slice.cpp +++ b/docs/template_plugin/tests/functional/op_reference/strided_slice.cpp @@ -368,6 +368,20 @@ std::vector generateStrideOptionalParams() { std::vector{0, 0, 0}, reference_tests::Tensor(IN_ET, {1, 4}, std::vector{20, 21, 22, 23}), "strided_slice_stride_optional_dynamic"), + // strided_slice_stride_optional_dynamic_empty_output_tensor + StridedSliceStrideOptionalParams( + PartialShape::dynamic(), + reference_tests::Tensor(IN_ET, {2, 3, 4}, std::vector{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23}), + reference_tests::Tensor(element::i64, {2}, std::vector{0, 0}), + reference_tests::Tensor(element::i64, {2}, std::vector{-1, 0}), + std::vector{1, 0}, + std::vector{1, 0}, + std::vector{}, + std::vector{}, + std::vector{}, + reference_tests::Tensor(IN_ET, {2, 0, 4}, std::vector{}), + "strided_slice_stride_optional_dynamic_empty_output_tensor"), }; return params; } diff --git a/src/core/reference/src/runtime/reference/strided_slice.cpp b/src/core/reference/src/runtime/reference/strided_slice.cpp index dd02f6412fb..e039e4d0917 100644 --- a/src/core/reference/src/runtime/reference/strided_slice.cpp +++ b/src/core/reference/src/runtime/reference/strided_slice.cpp @@ -18,6 +18,15 @@ void runtime::reference::strided_slice(const char* arg, const Shape& arg_shape, const SlicePlan& sp, size_t elem_type) { + auto hasZeroDims = [](const ov::Shape& shape) -> bool { + return std::any_of(shape.begin(), shape.end(), [](const size_t& dim) { + return dim == 0; + }); + }; + if (hasZeroDims(sp.reshape_in_shape) || hasZeroDims(sp.reshape_out_shape)) { + return; + } + runtime::AlignedBuffer slice_out_buffer(shape_size(sp.reshape_in_shape) * elem_type); slice(reinterpret_cast(arg), slice_out_buffer.get_ptr(), From 1efb0a034f3cae646e2357ca138730e8b89e60da Mon Sep 17 00:00:00 2001 From: Alexey Lebedev Date: Thu, 31 Mar 2022 16:12:48 +0300 Subject: [PATCH 33/39] [PYTHON API] release GIL (#10810) * AsyncInferQueue nogil update + refactoring * nogil in compiled model * nogil in Core * fix refactoring * nogil in infer_request * add tests * Fix code style * update test with incrementing reference counting * try to fix code style * fix code style * release gil in reshape and preprocessing * make args optional in test * fix code style * add docs about GIL * try to link doc string with docs * Apply suggestions from code review Co-authored-by: Jan Iwaszkiewicz * Fix docs * docs refactoring * Apply review comments * Fix code style Co-authored-by: Jan Iwaszkiewicz Co-authored-by: Anastasia Kuporosova --- docs/OV_Runtime_UG/Python_API_exclusives.md | 33 +++ docs/snippets/ov_python_exclusives.py | 36 +++ .../src/pyopenvino/core/async_infer_queue.cpp | 84 +++---- .../src/pyopenvino/core/compiled_model.cpp | 19 +- .../python/src/pyopenvino/core/core.cpp | 50 ++++- .../src/pyopenvino/core/infer_request.cpp | 27 ++- .../python/src/pyopenvino/graph/model.cpp | 31 ++- .../graph/preprocess/pre_post_process.cpp | 2 +- .../tests/test_inference_engine/test_nogil.py | 206 ++++++++++++++++++ 9 files changed, 421 insertions(+), 67 deletions(-) create mode 100644 src/bindings/python/tests/test_inference_engine/test_nogil.py diff --git a/docs/OV_Runtime_UG/Python_API_exclusives.md b/docs/OV_Runtime_UG/Python_API_exclusives.md index 92e9fa0266d..21e134ee6e4 100644 --- a/docs/OV_Runtime_UG/Python_API_exclusives.md +++ b/docs/OV_Runtime_UG/Python_API_exclusives.md @@ -75,3 +75,36 @@ Another feature of `AsyncInferQueue` is ability of setting callbacks. When callb The callback of `AsyncInferQueue` is uniform for every job. When executed, GIL is acquired to ensure safety of data manipulation inside the function. @snippet docs/snippets/ov_python_exclusives.py asyncinferqueue_set_callback + + +### Releasing the GIL + +Some functions in Python API release the Global Lock Interpreter (GIL) while running work-intensive code. It can help you to achieve more parallelism in your application using Python threads. For more information about GIL please refer to the Python documentation. + +@snippet docs/snippets/ov_python_exclusives.py releasing_gil + +> **NOTE**: While GIL is released functions can still modify and/or operate on Python objects in C++, thus there is no reference counting. User is responsible for thread safety if sharing of these objects with other thread occurs. It can affects your code only if multiple threads are spawned in Python.: + +#### List of functions that release the GIL +- openvino.runtime.AsyncInferQueue.start_async +- openvino.runtime.AsyncInferQueue.is_ready +- openvino.runtime.AsyncInferQueue.wait_all +- openvino.runtime.AsyncInferQueue.get_idle_request_id +- openvino.runtime.CompiledModel.create_infer_request +- openvino.runtime.CompiledModel.infer_new_request +- openvino.runtime.CompiledModel.__call__ +- openvino.runtime.CompiledModel.export +- openvino.runtime.CompiledModel.get_runtime_model +- openvino.runtime.Core.compile_model +- openvino.runtime.Core.read_model +- openvino.runtime.Core.import_model +- openvino.runtime.Core.query_model +- openvino.runtime.Core.get_available_devices +- openvino.runtime.InferRequest.infer +- openvino.runtime.InferRequest.start_async +- openvino.runtime.InferRequest.wait +- openvino.runtime.InferRequest.wait_for +- openvino.runtime.InferRequest.get_profiling_info +- openvino.runtime.InferRequest.query_state +- openvino.runtime.Model.reshape +- openvino.preprocess.PrePostProcessor.build diff --git a/docs/snippets/ov_python_exclusives.py b/docs/snippets/ov_python_exclusives.py index ecdb58e7a9b..6481ba6d5e9 100644 --- a/docs/snippets/ov_python_exclusives.py +++ b/docs/snippets/ov_python_exclusives.py @@ -131,3 +131,39 @@ infer_queue.wait_all() assert all(data_done) #! [asyncinferqueue_set_callback] + +#! [releasing_gil] +import openvino.runtime as ov +import cv2 as cv +from threading import Thread + +input_data = [] + +# Processing input data will be done in a separate thread +# while compilation of the model and creation of the infer request +# is going to be executed in the main thread. +def prepare_data(input, image_path): + image = cv.imread(image_path) + h, w = list(input.shape)[-2:] + image = cv.resize(image, (h, w)) + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, 0) + input_data.append(image) + +core = ov.Core() +model = core.read_model("model.xml") +# Create thread with prepare_data function as target and start it +thread = Thread(target=prepare_data, args=[model.input(), "path/to/image"]) +thread.start() +# The GIL will be released in compile_model. +# It allows a thread above to start the job, +# while main thread is running in the background. +compiled = core.compile_model(model, "GPU") +# After returning from compile_model, the main thread acquires the GIL +# and starts create_infer_request which releases it once again. +request = compiled.create_infer_request() +# Join the thread to make sure the input_data is ready +thread.join() +# running the inference +request.infer(input_data) +#! [releasing_gil] diff --git a/src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp b/src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp index c6438228ea1..09942edf63d 100644 --- a/src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp +++ b/src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp @@ -147,7 +147,6 @@ void regclass_AsyncInferQueue(py::module m) { if (jobs == 0) { jobs = (size_t)Common::get_optimal_number_of_requests(model); } - std::vector requests; std::queue idle_handles; std::vector user_ids(jobs); @@ -161,7 +160,6 @@ void regclass_AsyncInferQueue(py::module m) { requests.push_back(request); idle_handles.push(handle); } - return new AsyncInferQueue(requests, idle_handles, user_ids); }), py::arg("model"), @@ -212,71 +210,61 @@ void regclass_AsyncInferQueue(py::module m) { :type inputs: dict[Union[int, str, openvino.runtime.ConstOutput] : openvino.runtime.Tensor] :param userdata: Any data that will be passed to a callback :rtype: None + + GIL is released while waiting for the next available InferRequest. )"); - cls.def( - "is_ready", - [](AsyncInferQueue& self) { - return self._is_ready(); - }, - R"( + cls.def("is_ready", + &AsyncInferQueue::_is_ready, + R"( One of 'flow control' functions. Returns True if any free request in the pool, otherwise False. - Function releases GIL, other threads can work while this function waits. + GIL is released while running this function. :return: If there is at least one free InferRequest in a pool, returns True. :rtype: bool )"); - cls.def( - "wait_all", - [](AsyncInferQueue& self) { - return self.wait_all(); - }, - R"( - One of 'flow control' functions. Blocking call. - Waits for all InferRequests in a pool to finish scheduled work. + cls.def("wait_all", + &AsyncInferQueue::wait_all, + R"( + One of 'flow control' functions. Blocking call. + Waits for all InferRequests in a pool to finish scheduled work. - Function releases GIL, other threads can work while this function waits. - )"); + GIL is released while running this function. + )"); - cls.def( - "get_idle_request_id", - [](AsyncInferQueue& self) { - return self.get_idle_request_id(); - }, - R"( - Returns next free id of InferRequest from queue's pool. - Function waits for any request to complete and then returns this request's id. + cls.def("get_idle_request_id", + &AsyncInferQueue::get_idle_request_id, + R"( + Returns next free id of InferRequest from queue's pool. + Function waits for any request to complete and then returns this request's id. - Function releases GIL, other threads can work while this function waits. + GIL is released while running this function. - :rtype: int - )"); + :rtype: int + )"); - cls.def( - "set_callback", - [](AsyncInferQueue& self, py::function callback) { - self.set_custom_callbacks(callback); - }, - R"( - Sets unified callback on all InferRequests from queue's pool. - The signature of such function should have two arguments, where - the first one is InferRequest object and the second one is userdata - connected to InferRequest from the AsyncInferQueue's pool. + cls.def("set_callback", + &AsyncInferQueue::set_custom_callbacks, + R"( + Sets unified callback on all InferRequests from queue's pool. + Signature of such function should have two arguments, where + first one is InferRequest object and second one is userdata + connected to InferRequest from the AsyncInferQueue's pool. - .. code-block:: python + .. code-block:: python - def f(request, userdata): - result = request.output_tensors[0] - print(result + userdata) + def f(request, userdata): + result = request.output_tensors[0] + print(result + userdata) - async_infer_queue.set_callback(f) + async_infer_queue.set_callback(f) - :param callback: Any Python defined function that matches callback's requirements. - :type callback: function - )"); + :param callback: Any Python defined function that matches callback's requirements. + :type callback: function + )"); cls.def( "__len__", diff --git a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp index 768768a1872..c56a545f2d3 100644 --- a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp +++ b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp @@ -33,6 +33,7 @@ void regclass_CompiledModel(py::module m) { [](ov::CompiledModel& self) { return std::make_shared(self.create_infer_request(), self.inputs(), self.outputs()); }, + py::call_guard(), R"( Creates an inference request object used to infer the compiled model. The created request has allocated input and output tensors. @@ -47,7 +48,10 @@ void regclass_CompiledModel(py::module m) { auto request = self.create_infer_request(); // Update inputs if there are any Common::set_request_tensors(request, inputs); - request.infer(); + { + py::gil_scoped_release release; + request.infer(); + } return Common::outputs_to_dict(self.outputs(), request); }, py::arg("inputs"), @@ -59,6 +63,8 @@ void regclass_CompiledModel(py::module m) { It is advised to use a dedicated InferRequest class for performance, optimizing workflows, and creating advanced pipelines. + GIL is released during the inference. + :param inputs: Data to set on input tensors. :type inputs: Dict[Union[int, str, openvino.runtime.ConstOutput], openvino.runtime.Tensor] :return: Dictionary of results from output tensors with ports as keys. @@ -72,9 +78,12 @@ void regclass_CompiledModel(py::module m) { self.export_model(_stream); return py::bytes(_stream.str()); }, + py::call_guard(), R"( Exports the compiled model to bytes/output stream. + GIL is released while running this function. + :return: Bytes object that contains this compiled model. :rtype: bytes @@ -99,7 +108,10 @@ void regclass_CompiledModel(py::module m) { (std::string)(py::repr(model_stream)) + "` provided"); } std::stringstream _stream; - self.export_model(_stream); + { + py::gil_scoped_release release; + self.export_model(_stream); + } model_stream.attr("flush")(); model_stream.attr("write")(py::bytes(_stream.str())); model_stream.attr("seek")(0); // Always rewind stream! @@ -114,6 +126,8 @@ void regclass_CompiledModel(py::module m) { Function performs flushing of the stream, writes to it, and then rewinds the stream to the beginning (using seek(0)). + GIL is released while running this function. + :param model_stream: A stream object to which the model will be serialized. :type model_stream: io.BytesIO :rtype: None @@ -165,6 +179,7 @@ void regclass_CompiledModel(py::module m) { cls.def("get_runtime_model", &ov::CompiledModel::get_runtime_model, + py::call_guard(), R"( Gets runtime model information from a device. diff --git a/src/bindings/python/src/pyopenvino/core/core.cpp b/src/bindings/python/src/pyopenvino/core/core.cpp index 7163bbe129b..866315052a3 100644 --- a/src/bindings/python/src/pyopenvino/core/core.cpp +++ b/src/bindings/python/src/pyopenvino/core/core.cpp @@ -77,6 +77,7 @@ void regclass_Core(py::module m) { const std::map& properties) { return self.compile_model(model, device_name, {properties.begin(), properties.end()}); }, + py::call_guard(), py::arg("model"), py::arg("device_name"), py::arg("config") = py::dict(), @@ -85,6 +86,8 @@ void regclass_Core(py::module m) { Users can create as many compiled models as they need, and use them simultaneously (up to the limitation of the hardware resources). + GIL is released while running this function. + :param model: Model acquired from read_model function. :type model: openvino.runtime.Model :param device_name: Name of the device which will load the model. @@ -102,6 +105,7 @@ void regclass_Core(py::module m) { const std::map& config) { return self.compile_model(model, ov::AnyMap{config.begin(), config.end()}); }, + py::call_guard(), py::arg("model"), py::arg("config") = py::dict(), R"( @@ -109,6 +113,8 @@ void regclass_Core(py::module m) { selected by AUTO plugin. Users can create as many compiled models as they need, and use them simultaneously (up to the limitation of the hardware resources). + GIL is released while running this function. + :param model: Model acquired from read_model function. :type model: openvino.runtime.Model :param properties: Optional dict of pairs: (property name, property value) relevant only for this load operation. @@ -125,6 +131,7 @@ void regclass_Core(py::module m) { const std::map& config) { return self.compile_model(model_path, device_name, {config.begin(), config.end()}); }, + py::call_guard(), py::arg("model_path"), py::arg("device_name"), py::arg("properties") = py::dict(), @@ -133,6 +140,8 @@ void regclass_Core(py::module m) { This can be more efficient than using read_model + compile_model(model_in_memory_object) flow, especially for cases when caching is enabled and cached model is available. + GIL is released while running this function. + :param model_path: A path to a model in IR / ONNX / PDPD format. :type model_path: str :param device_name: Name of the device to load the model to. @@ -148,6 +157,7 @@ void regclass_Core(py::module m) { [](ov::Core& self, const std::string& model_path, const std::map& properties) { return self.compile_model(model_path, ov::AnyMap{properties.begin(), properties.end()}); }, + py::call_guard(), py::arg("model_path"), py::arg("config") = py::dict(), R"( @@ -155,6 +165,8 @@ void regclass_Core(py::module m) { This can be more efficient than using read_model + compile_model(model_in_memory_object) flow, especially for cases when caching is enabled and cached model is available. + GIL is released while running this function. + :param model_path: A path to a model in IR / ONNX / PDPD format. :type model_path: str :param properties: Optional dict of pairs: (property name, property value) relevant only for this load operation. @@ -178,25 +190,26 @@ void regclass_Core(py::module m) { cls.def( "read_model", [](ov::Core& self, py::bytes model, py::bytes weights) { + std::string ir(model); // works on view in order to omit copying bytes into string py::buffer_info info(py::buffer(weights).request()); size_t bin_size = static_cast(info.size); + ov::Tensor tensor(ov::element::Type_t::u8, {bin_size}); // if weights are not empty if (bin_size) { const uint8_t* bin = reinterpret_cast(info.ptr); - ov::Tensor tensor(ov::element::Type_t::u8, {bin_size}); std::memcpy(tensor.data(), bin, bin_size); - return self.read_model(model, tensor); } - // create empty tensor of type u8 - ov::Tensor tensor(ov::element::Type_t::u8, {}); - return self.read_model(model, tensor); + py::gil_scoped_release release; + return self.read_model(ir, tensor); }, py::arg("model"), py::arg("weights") = py::bytes(), R"( Reads models from IR / ONNX / PDPD formats. + GIL is released while running this function. + :param model: Bytes with model in IR / ONNX / PDPD format. :type model: bytes :param weights: Bytes with tensor's data. @@ -208,11 +221,14 @@ void regclass_Core(py::module m) { cls.def( "read_model", (std::shared_ptr(ov::Core::*)(const std::string&, const std::string&) const) & ov::Core::read_model, + py::call_guard(), py::arg("model"), py::arg("weights") = "", R"( Reads models from IR / ONNX / PDPD formats. + GIL is released while running this function. + :param model: A path to a model in IR / ONNX / PDPD format. :type model: str :param weights: A path to a data file For IR format (*.bin): if path is empty, @@ -228,11 +244,14 @@ void regclass_Core(py::module m) { cls.def( "read_model", (std::shared_ptr(ov::Core::*)(const std::string&, const ov::Tensor&) const) & ov::Core::read_model, + py::call_guard(), py::arg("model"), py::arg("weights"), R"( Reads models from IR / ONNX / PDPD formats. + GIL is released while running this function. + :param model: A string with model in IR / ONNX / PDPD format. :type model: str :param weights: Tensor with weights. Reading ONNX / PDPD models doesn't support @@ -244,14 +263,19 @@ void regclass_Core(py::module m) { cls.def( "read_model", - [](ov::Core& self, py::object model, py::object weights) { - return self.read_model(py::str(model), py::str(weights)); + [](ov::Core& self, py::object model_path, py::object weights_path) { + std::string model_path_cpp{py::str(model_path)}; + std::string weights_path_cpp{py::str(weights_path)}; + py::gil_scoped_release release; + return self.read_model(model_path_cpp, weights_path_cpp); }, py::arg("model"), py::arg("weights") = "", R"( Reads models from IR / ONNX / PDPD formats. + GIL is released while running this function. + :param model: A string with model in IR / ONNX / PDPD format. :type model: str :param weights: A path to a data file For IR format (*.bin): if path is empty, @@ -273,12 +297,15 @@ void regclass_Core(py::module m) { _stream << model_stream; return self.import_model(_stream, device_name, {properties.begin(), properties.end()}); }, + py::call_guard(), py::arg("model_stream"), py::arg("device_name"), py::arg("properties") = py::none(), R"( Imports a compiled model from a previously exported one. + GIL is released while running this function. + :param model_stream: Input stream, containing a model previously exported, using export_model method. :type model_stream: bytes :param device_name: Name of device to which compiled model is imported. @@ -319,6 +346,7 @@ void regclass_Core(py::module m) { _stream << model_stream .attr("read")() // alternative: model_stream.attr("get_value")() .cast(); + py::gil_scoped_release release; return self.import_model(_stream, device_name, {properties.begin(), properties.end()}); }, py::arg("model_stream"), @@ -330,6 +358,8 @@ void regclass_Core(py::module m) { Advanced version of `import_model`. It utilizes, streams from standard Python library `io`. + GIL is released while running this function. + :param model_stream: Input stream, containing a model previously exported, using export_model method. :type model_stream: io.BytesIO @@ -419,12 +449,15 @@ void regclass_Core(py::module m) { const std::map& properties) { return self.query_model(model, device_name, {properties.begin(), properties.end()}); }, + py::call_guard(), py::arg("model"), py::arg("device_name"), py::arg("properties") = py::dict(), R"( Query device if it supports specified model with specified properties. + GIL is released while running this function. + :param model: Model object to query. :type model: openvino.runtime.Model :param device_name: A name of a device to query. @@ -468,9 +501,12 @@ void regclass_Core(py::module m) { cls.def_property_readonly("available_devices", &ov::Core::get_available_devices, + py::call_guard(), R"( Returns devices available for inference Core objects goes over all registered plugins. + GIL is released while running this function. + :returns: A list of devices. The devices are returned as: CPU, GPU.0, GPU.1, MYRIAD... If there more than one device of specific type, they are enumerated with .# suffix. Such enumerated device can later be used as a device name in all Core methods like: diff --git a/src/bindings/python/src/pyopenvino/core/infer_request.cpp b/src/bindings/python/src/pyopenvino/core/infer_request.cpp index 8d70c7bd23c..82864f4fd27 100644 --- a/src/bindings/python/src/pyopenvino/core/infer_request.cpp +++ b/src/bindings/python/src/pyopenvino/core/infer_request.cpp @@ -164,9 +164,12 @@ void regclass_InferRequest(py::module m) { // Update inputs if there are any Common::set_request_tensors(self._request, inputs); // Call Infer function - self._start_time = Time::now(); - self._request.infer(); - self._end_time = Time::now(); + { + py::gil_scoped_release release; + self._start_time = Time::now(); + self._request.infer(); + self._end_time = Time::now(); + } return Common::outputs_to_dict(self._outputs, self._request); }, py::arg("inputs"), @@ -175,6 +178,8 @@ void regclass_InferRequest(py::module m) { Blocks all methods of InferRequest while request is running. Calling any method will lead to throwing exceptions. + GIL is released while running the inference. + :param inputs: Data to set on input tensors. :type inputs: Dict[Union[int, str, openvino.runtime.ConstOutput], openvino.runtime.Tensor] :return: Dictionary of results from output tensors with ports as keys. @@ -203,8 +208,7 @@ void regclass_InferRequest(py::module m) { Starts inference of specified input(s) in asynchronous mode. Returns immediately. Inference starts also immediately. - This function releases the GIL, so another Python thread can - work while this function runs in the background. + GIL is released while running the inference. Calling any method on this InferRequest while the request is running will lead to throwing exceptions. @@ -234,7 +238,7 @@ void regclass_InferRequest(py::module m) { Waits for the result to become available. Blocks until the result becomes available. - Function releases GIL, other threads can work while this function waits. + GIL is released while running this function. )"); cls.def( @@ -249,7 +253,7 @@ void regclass_InferRequest(py::module m) { Blocks until specified timeout has elapsed or the result becomes available, whichever comes first. - Function releases GIL, other threads can work while this function waits. + GIL is released while running this function. :param timeout: Maximum duration in milliseconds (ms) of blocking call. :type timeout: int @@ -512,11 +516,14 @@ void regclass_InferRequest(py::module m) { [](InferRequestWrapper& self) { return self._request.get_profiling_info(); }, + py::call_guard(), R"( Queries performance is measured per layer to get feedback on what is the most time-consuming operation, not all plugins provide meaningful data. + GIL is released while running this function. + :return: List of profiling information for operations in model. :rtype: List[openvino.runtime.ProfilingInfo] )"); @@ -526,9 +533,12 @@ void regclass_InferRequest(py::module m) { [](InferRequestWrapper& self) { return self._request.query_state(); }, + py::call_guard(), R"( Gets state control interface for given infer request. + GIL is released while running this function. + :return: List of VariableState objects. :rtype: List[openvino.runtime.VariableState] )"); @@ -615,9 +625,12 @@ void regclass_InferRequest(py::module m) { [](InferRequestWrapper& self) { return self._request.get_profiling_info(); }, + py::call_guard(), R"( Performance is measured per layer to get feedback on the most time-consuming operation. Not all plugins provide meaningful data! + + GIL is released while running this function. :return: Inference time. :rtype: List[openvino.runtime.ProfilingInfo] diff --git a/src/bindings/python/src/pyopenvino/graph/model.cpp b/src/bindings/python/src/pyopenvino/graph/model.cpp index ab76a678623..25e90d7dbf1 100644 --- a/src/bindings/python/src/pyopenvino/graph/model.cpp +++ b/src/bindings/python/src/pyopenvino/graph/model.cpp @@ -263,8 +263,13 @@ void regclass_graph_Model(py::module m) { [](ov::Model& self, const ov::PartialShape& partial_shape) { self.reshape(partial_shape); }, + py::call_guard(), py::arg("partial_shape"), R"( + Reshape model input. + + GIL is released while running this function. + :param partial_shape: New shape. :type partial_shape: openvino.runtime.PartialShape :return : void @@ -273,10 +278,16 @@ void regclass_graph_Model(py::module m) { model.def( "reshape", [](ov::Model& self, const py::list& partial_shape) { - self.reshape(Common::partial_shape_from_list(partial_shape)); + ov::PartialShape new_shape(Common::partial_shape_from_list(partial_shape)); + py::gil_scoped_release release; + self.reshape(new_shape); }, py::arg("partial_shape"), R"( + Reshape model input. + + GIL is released while running this function. + :param partial_shape: New shape. :type partial_shape: list :return : void @@ -285,10 +296,16 @@ void regclass_graph_Model(py::module m) { model.def( "reshape", [](ov::Model& self, const py::tuple& partial_shape) { - self.reshape(Common::partial_shape_from_list(partial_shape.cast())); + ov::PartialShape new_shape(Common::partial_shape_from_list(partial_shape.cast())); + py::gil_scoped_release release; + self.reshape(new_shape); }, py::arg("partial_shape"), R"( + Reshape model input. + + GIL is released while running this function. + :param partial_shape: New shape. :type partial_shape: tuple :return : void @@ -299,8 +316,13 @@ void regclass_graph_Model(py::module m) { [](ov::Model& self, const std::string& partial_shape) { self.reshape(Common::partial_shape_from_str(partial_shape)); }, + py::call_guard(), py::arg("partial_shape"), R"( + Reshape model input. + + GIL is released while running this function. + :param partial_shape: New shape. :type partial_shape: str :return : void @@ -337,6 +359,7 @@ void regclass_graph_Model(py::module m) { } new_shapes.insert(new_shape); } + py::gil_scoped_release release; self.reshape(new_shapes); }, py::arg("partial_shapes"), @@ -368,6 +391,10 @@ void regclass_graph_Model(py::module m) { '1..10' - to define bounded dimension '..10' or '1..' to define dimension with only lower or only upper limit + Reshape model input. + + GIL is released while running this function. + :param partial_shapes: New shapes. :type partial_shapes: Dict[keys, values] )"); diff --git a/src/bindings/python/src/pyopenvino/graph/preprocess/pre_post_process.cpp b/src/bindings/python/src/pyopenvino/graph/preprocess/pre_post_process.cpp index 11af911b0dc..d5ba05d8733 100644 --- a/src/bindings/python/src/pyopenvino/graph/preprocess/pre_post_process.cpp +++ b/src/bindings/python/src/pyopenvino/graph/preprocess/pre_post_process.cpp @@ -480,7 +480,7 @@ void regclass_graph_PrePostProcessor(py::module m) { }, py::arg("output_index")); - proc.def("build", &ov::preprocess::PrePostProcessor::build); + proc.def("build", &ov::preprocess::PrePostProcessor::build, py::call_guard()); proc.def("__str__", [](const ov::preprocess::PrePostProcessor& self) -> std::string { std::stringstream ss; diff --git a/src/bindings/python/tests/test_inference_engine/test_nogil.py b/src/bindings/python/tests/test_inference_engine/test_nogil.py new file mode 100644 index 00000000000..d2783f02bb0 --- /dev/null +++ b/src/bindings/python/tests/test_inference_engine/test_nogil.py @@ -0,0 +1,206 @@ +# Copyright (C) 2018-2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import time +import pytest +import os +import io +from threading import Thread +import numpy as np + +from openvino.runtime import Core, Model, AsyncInferQueue, PartialShape, Layout, opset8 as ops, serialize +from openvino.preprocess import PrePostProcessor + + +# check if func releases the GIL and doens't increment reference counters of args while GIL is released +def check_gil_released_safe(func, args=[]): # noqa: B006 + global gil_released + gil_released = False + + def detect_gil(): + global gil_released + # while sleeping main thread acquires GIL and runs func, which will release GIL + time.sleep(0.000001) + # increment reference counting of args while running func + args_ = args # noqa: F841 'assigned to but never used' + gil_released = True + thread = Thread(target=detect_gil) + thread.start() + func(*args) + if not gil_released: + pytest.xfail(reason="Depend on condition race") + thread.join() + + +device = os.environ.get("TEST_DEVICE") if os.environ.get("TEST_DEVICE") else "CPU" +core = Core() +core.set_property({"PERF_COUNT": "YES"}) +param = ops.parameter([224, 224]) +model = Model(ops.relu(param), [param]) +compiled = core.compile_model(model, device) +infer_queue = AsyncInferQueue(compiled, 1) +user_stream = io.BytesIO() + + +# AsyncInferQueue + +def test_gil_released_async_infer_queue_start_async(): + infer_queue.start_async() + check_gil_released_safe(infer_queue.start_async) + + +def test_gil_released_async_infer_queue_is_ready(): + infer_queue.start_async() + check_gil_released_safe(infer_queue.is_ready) + + +def test_gil_released_async_infer_queue_wait_all(): + infer_queue.start_async() + check_gil_released_safe(infer_queue.wait_all) + + +def test_gil_released_async_infer_queue_get_idle_request_id(): + infer_queue.start_async() + check_gil_released_safe(infer_queue.get_idle_request_id) + + +# CompiledModel + +def test_gil_released_create_infer_request(): + check_gil_released_safe(compiled.create_infer_request) + + +def test_gil_released_infer_new_request(): + check_gil_released_safe(compiled) + + +def test_gil_released_export(): + check_gil_released_safe(compiled.export_model) + + +def test_gil_released_export_advanced(): + check_gil_released_safe(compiled.export_model, [user_stream]) + + +def test_gil_released_get_runtime_model(): + check_gil_released_safe(compiled.get_runtime_model) + + +# Core + +def test_compile_model(device): + check_gil_released_safe(core.compile_model, [model, device]) + + +def test_read_model_from_bytes(): + ir = bytes(b""" + + + + + + 10 + + + + + + + 10 + + + + + 10 + + + + + + + 10 + + + + + + + + +""") + check_gil_released_safe(core.read_model, [ir]) + + +def test_read_model_from_path(): + from pathlib import Path + model_path = "relu.xml" + bin_path = "relu.bin" + serialize(model, model_path, bin_path) + check_gil_released_safe(core.read_model, [Path(model_path)]) + os.remove(model_path) + os.remove(bin_path) + + +def test_import_model(device): + check_gil_released_safe(core.import_model, [user_stream, device]) + + +def test_query_model(device): + check_gil_released_safe(core.query_model, [model, device]) + + +def test_get_available_devices(device): + check_gil_released_safe(getattr, [core, "available_devices"]) + + +# InferRequest + +request = compiled.create_infer_request() + + +def test_infer(): + data = [np.random.normal(size=list(compiled.input().shape))] + check_gil_released_safe(request.infer, [data]) + + +def test_start_async(): + data = [np.random.normal(size=list(compiled.input().shape))] + check_gil_released_safe(request.start_async, [data]) + request.wait() + + +def test_wait(): + data = [np.random.normal(size=list(compiled.input().shape))] + request.start_async(data) + check_gil_released_safe(request.wait) + + +def test_wait_for(): + data = [np.random.normal(size=list(compiled.input().shape))] + request.start_async(data) + check_gil_released_safe(request.wait_for, [1]) + request.wait() + + +def test_get_profiling_info(): + check_gil_released_safe(request.get_profiling_info) + + +def test_query_state(): + check_gil_released_safe(request.query_state) + + +# Preprocessing + +def test_pre_post_process_build(): + p = PrePostProcessor(model) + p.input().model().set_layout(Layout("NC")) + check_gil_released_safe(p.build) + + +def test_model_reshape(): + check_gil_released_safe(model.reshape, [PartialShape([128, 128])]) + check_gil_released_safe(model.reshape, [[164, 164]]) + check_gil_released_safe(model.reshape, [(178, 178)]) + check_gil_released_safe(model.reshape, ["194, 194"]) + check_gil_released_safe(model.reshape, [{0: [224, 224]}]) From 15b4553eafbc7ca88204a07d21391917f2f58df6 Mon Sep 17 00:00:00 2001 From: Vladimir Paramuzov Date: Thu, 31 Mar 2022 17:13:49 +0300 Subject: [PATCH 34/39] [GPU] Align OneHot primitive parameters with ngraph (#11361) --- .../include/intel_gpu/primitives/one_hot.hpp | 26 ++++++--- .../intel_gpu/src/graph/impls/ocl/one_hot.cpp | 4 +- .../intel_gpu/src/plugin/ops/one_hot.cpp | 22 ++++++-- .../tests/test_cases/one_hot_gpu_test.cpp | 56 +++++++++---------- 4 files changed, 63 insertions(+), 45 deletions(-) diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/one_hot.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/one_hot.hpp index 67fb2342101..38409948e13 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/primitives/one_hot.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/primitives/one_hot.hpp @@ -49,13 +49,18 @@ struct one_hot : public primitive_base { one_hot(const primitive_id& id, const primitive_id& input, const tensor& shape, - const uint16_t& one_hot_axis, + const int64_t& one_hot_axis, + const int64_t& depth, const float& on_value = 1.0f, const float& off_value = 0.0f, const primitive_id& ext_prim_id = "", const padding& output_padding = padding()) - : primitive_base(id, {input}, ext_prim_id, output_padding), shape(shape), one_hot_axis(one_hot_axis), - on_value(on_value), off_value(off_value) {} + : primitive_base(id, {input}, ext_prim_id, output_padding) + , shape(shape) + , one_hot_axis(one_hot_axis) + , depth(depth) + , on_value(on_value) + , off_value(off_value) {} /// @brief Constructs one-hot primitive layer. /// @param id An identifier of new primitive. @@ -68,18 +73,25 @@ struct one_hot : public primitive_base { const primitive_id& input, const tensor& shape, const data_types output_dt, - const uint16_t& one_hot_axis, + const int64_t& one_hot_axis, + const int64_t& depth, const float& on_value = 1.0f, const float& off_value = 0.0f, const primitive_id& ext_prim_id = "", const padding& output_padding = padding()) - : primitive_base(id, {input}, ext_prim_id, output_padding, optional_data_type{output_dt}), shape(shape), one_hot_axis(one_hot_axis), - on_value(on_value), off_value(off_value) {} + : primitive_base(id, {input}, ext_prim_id, output_padding, optional_data_type{output_dt}) + , shape(shape) + , one_hot_axis(one_hot_axis) + , depth(depth) + , on_value(on_value) + , off_value(off_value) {} /// @brief Output size reference. tensor shape; /// @brief One-hot axis position in output shape (0-based, from left to right). - uint16_t one_hot_axis; + int64_t one_hot_axis; + /// @brief The number of classes and thus the size of the one-hot dimension + int64_t depth; /// @brief The locations represented by indices in indices take this value. float on_value; /// @brief all other locations take value this value. diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/one_hot.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/one_hot.cpp index a5097bf836e..482e1b16c70 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/one_hot.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/one_hot.cpp @@ -32,9 +32,7 @@ struct one_hot_impl : typed_primitive_impl_ocl { oh_params.on_value = arg.get_primitive()->on_value; oh_params.off_value = arg.get_primitive()->off_value; - auto output_sizes = arg.get_output_layout().format == format::bfzyx ? - arg.get_output_layout().size.sizes(format::bfzyx) : - arg.get_output_layout().size.sizes(format::bfyx); + auto output_sizes = arg.get_output_layout().get_dims(); oh_params.one_hot_limit = output_sizes[oh_params.one_hot_axis]; diff --git a/src/plugins/intel_gpu/src/plugin/ops/one_hot.cpp b/src/plugins/intel_gpu/src/plugin/ops/one_hot.cpp index 861f0edd537..41a2d029057 100644 --- a/src/plugins/intel_gpu/src/plugin/ops/one_hot.cpp +++ b/src/plugins/intel_gpu/src/plugin/ops/one_hot.cpp @@ -19,12 +19,13 @@ static void CreateOneHotOp(Program& p, const std::shared_ptrget_axis(); + int64_t axis = op->get_axis(); + auto depth_value_node = std::dynamic_pointer_cast(op->get_input_node_shared_ptr(1)); auto on_value_node = std::dynamic_pointer_cast(op->get_input_node_shared_ptr(2)); auto off_value_node = std::dynamic_pointer_cast(op->get_input_node_shared_ptr(3)); - if (on_value_node == nullptr || off_value_node == nullptr) - IE_THROW() << "Unsupported on/off node type in " << op->get_friendly_name() << " (" << op->get_type_name() << ")"; + if (on_value_node == nullptr || off_value_node == nullptr || depth_value_node == nullptr) + IE_THROW() << "Unsupported on/off/depth node type in " << op->get_friendly_name() << " (" << op->get_type_name() << ")"; float on_value; float off_value; @@ -34,7 +35,7 @@ static void CreateOneHotOp(Program& p, const std::shared_ptrget_friendly_name() << " (" << op->get_type_name() << ")"; } - auto dims = op->get_input_shape(0); + auto dims = op->get_input_partial_shape(0); if (axis < -1 || axis > static_cast(dims.size())) IE_THROW() << op->get_friendly_name() << " Incorrect OneHot axis value: " << axis << ". Should be between -1 and " << dims.size(); @@ -49,11 +50,20 @@ static void CreateOneHotOp(Program& p, const std::shared_ptrcast_vector()[0]; + + auto out_pshape = op->get_output_partial_shape(0); + if (out_pshape.is_dynamic()) { + IE_THROW() << "OneHot doesn't support dynamic shapes yet"; + } + auto out_tensor = tensor_from_dims(out_pshape.to_shape()); + auto oneHotPrim = cldnn::one_hot(layerName, inputPrimitives[0], - tensor_from_dims(op->get_output_shape(0)), + out_tensor, DataTypeFromPrecision(op->get_output_element_type(0)), - static_cast(axis), + axis, + depth, on_value, off_value, op->get_friendly_name()); diff --git a/src/plugins/intel_gpu/tests/test_cases/one_hot_gpu_test.cpp b/src/plugins/intel_gpu/tests/test_cases/one_hot_gpu_test.cpp index c18f4c2621b..cc48db1ca2d 100644 --- a/src/plugins/intel_gpu/tests/test_cases/one_hot_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/test_cases/one_hot_gpu_test.cpp @@ -84,7 +84,7 @@ void generic_one_hot_test_int(cldnn::format test_input_fmt, int input_b, int inp topology topology; topology.add(input_layout("input", input->get_layout())); - topology.add(one_hot("output", "input", shape, one_hot_axis)); + topology.add(one_hot("output", "input", shape, one_hot_axis, one_hot_limit)); network network(engine, topology); network.set_input_data("input", input); @@ -165,7 +165,7 @@ TEST(one_hot_gpu_i32, bfzyx_ax4) { auto input = engine.allocate_memory({ data_types::i32, format::bfyx, input_tensor }); topology topology; topology.add(input_layout("input", input->get_layout())); - topology.add(one_hot("output","input", shape, one_hot_axis)); + topology.add(one_hot("output","input", shape, one_hot_axis, 5)); set_values(input, input_rnd_vec); @@ -224,7 +224,7 @@ TEST(one_hot_gpu_i64, bfzyx_ax4) { auto input = engine.allocate_memory({ data_types::i64, format::bfyx, input_tensor }); topology topology; topology.add(input_layout("input", input->get_layout())); - topology.add(one_hot("output","input", shape, one_hot_axis)); + topology.add(one_hot("output","input", shape, one_hot_axis, 5)); set_values(input, input_rnd_vec); @@ -283,7 +283,7 @@ TEST(one_hot_gpu_i32_to_f32, bfyx_ax4) { auto input = engine.allocate_memory({ data_types::i32, format::bfyx, input_tensor }); topology topology; topology.add(input_layout("input", input->get_layout())); - topology.add(one_hot("output","input", shape, data_types::f32, one_hot_axis)); + topology.add(one_hot("output","input", shape, data_types::f32, one_hot_axis, 5)); set_values(input, input_rnd_vec); @@ -297,12 +297,11 @@ TEST(one_hot_gpu_i32_to_f32, bfyx_ax4) { auto output_layout = output_memory->get_layout(); cldnn::mem_lock output_ptr(output_memory, get_test_stream()); - tensor output_tensor = output_layout.get_buffer_size(); - int z_size = output_tensor.spatial[2]; - int y_size = output_tensor.spatial[1]; - int x_size = output_tensor.spatial[0]; - int f_size = output_tensor.feature[0]; - int b_size = output_tensor.batch[0]; + int z_size = output_layout.spatial(2); + int y_size = output_layout.spatial(1); + int x_size = output_layout.spatial(0); + int f_size = output_layout.feature(); + int b_size = output_layout.batch(); EXPECT_EQ(z_size, 2); EXPECT_EQ(y_size, 1); EXPECT_EQ(x_size, 5); @@ -337,7 +336,7 @@ TEST(one_hot_gpu_i64_to_f32, bfyx_ax4) { auto input = engine.allocate_memory({ data_types::i64, format::bfyx, input_tensor }); topology topology; topology.add(input_layout("input", input->get_layout())); - topology.add(one_hot("output","input", shape, data_types::f32, one_hot_axis)); + topology.add(one_hot("output","input", shape, data_types::f32, one_hot_axis, 5)); set_values(input, input_rnd_vec); @@ -388,7 +387,7 @@ TEST(one_hot_gpu_i32, bfzyx_ax0) { auto input = engine.allocate_memory({ data_types::i32, format::bfyx, input_tensor }); topology topology; topology.add(input_layout("input", input->get_layout())); - topology.add(one_hot("output","input", shape, one_hot_axis)); + topology.add(one_hot("output","input", shape, one_hot_axis, 3)); set_values(input, input_rnd_vec); @@ -443,7 +442,7 @@ TEST(one_hot_gpu_i64, bfzyx_ax0) { auto input = engine.allocate_memory({ data_types::i64, format::bfyx, input_tensor }); topology topology; topology.add(input_layout("input", input->get_layout())); - topology.add(one_hot("output","input", shape, one_hot_axis)); + topology.add(one_hot("output","input", shape, one_hot_axis, 3)); set_values(input, input_rnd_vec); @@ -498,7 +497,7 @@ TEST(one_hot_gpu_i32, bfzyx_ax1) { auto input = engine.allocate_memory({ data_types::i32, format::bfyx, input_tensor }); topology topology; topology.add(input_layout("input", input->get_layout())); - topology.add(one_hot("output","input", shape, one_hot_axis)); + topology.add(one_hot("output","input", shape, one_hot_axis, 3)); set_values(input, input_rnd_vec); @@ -553,7 +552,7 @@ TEST(one_hot_gpu_i64, bfzyx_ax1) { auto input = engine.allocate_memory({ data_types::i64, format::bfyx, input_tensor }); topology topology; topology.add(input_layout("input", input->get_layout())); - topology.add(one_hot("output","input", shape, one_hot_axis)); + topology.add(one_hot("output","input", shape, one_hot_axis, 3)); set_values(input, input_rnd_vec); @@ -608,7 +607,7 @@ TEST(one_hot_gpu_i32, bfzyx_ax2) { auto input = engine.allocate_memory({ data_types::i32, format::bfyx, input_tensor }); topology topology; topology.add(input_layout("input", input->get_layout())); - topology.add(one_hot("output","input", shape, one_hot_axis)); + topology.add(one_hot("output","input", shape, one_hot_axis, 3)); set_values(input, input_rnd_vec); @@ -663,7 +662,7 @@ TEST(one_hot_gpu_i64, bfzyx_ax2) { auto input = engine.allocate_memory({ data_types::i64, format::bfyx, input_tensor }); topology topology; topology.add(input_layout("input", input->get_layout())); - topology.add(one_hot("output","input", shape, one_hot_axis)); + topology.add(one_hot("output","input", shape, one_hot_axis, 3)); set_values(input, input_rnd_vec); @@ -718,7 +717,7 @@ TEST(one_hot_gpu_i32, bfzyx_ax3) { auto input = engine.allocate_memory({ data_types::i32, format::bfyx, input_tensor }); topology topology; topology.add(input_layout("input", input->get_layout())); - topology.add(one_hot("output","input", shape, one_hot_axis)); + topology.add(one_hot("output","input", shape, one_hot_axis, 3)); set_values(input, input_rnd_vec); @@ -773,7 +772,7 @@ TEST(one_hot_gpu_i64, bfzyx_ax3) { auto input = engine.allocate_memory({ data_types::i64, format::bfyx, input_tensor }); topology topology; topology.add(input_layout("input", input->get_layout())); - topology.add(one_hot("output","input", shape, one_hot_axis)); + topology.add(one_hot("output","input", shape, one_hot_axis, 3)); set_values(input, input_rnd_vec); @@ -787,12 +786,11 @@ TEST(one_hot_gpu_i64, bfzyx_ax3) { auto output_layout = output_memory->get_layout(); cldnn::mem_lock output_ptr(output_memory, get_test_stream()); - tensor output_tensor = output_layout.get_buffer_size(); - int z_size = output_tensor.spatial[2]; - int y_size = output_tensor.spatial[1]; - int x_size = output_tensor.spatial[0]; - int f_size = output_tensor.feature[0]; - int b_size = output_tensor.batch[0]; + int z_size = output_layout.spatial(2); + int y_size = output_layout.spatial(1); + int x_size = output_layout.spatial(0); + int f_size = output_layout.feature(); + int b_size = output_layout.batch(); EXPECT_EQ(z_size, 1); EXPECT_EQ(y_size, 3); EXPECT_EQ(x_size, 2); @@ -813,11 +811,11 @@ TEST(one_hot_gpu_i64, bfzyx_ax3) { TEST(one_hot_error, basic_error_wrong_axis) { auto& engine = get_test_engine(); - auto input = engine.allocate_memory({ data_types::i32, format::bfyx,{ 1, 1, 1, 1 } }); + auto input = engine.allocate_memory({ data_types::i32, format::bfyx, tensor{ 1, 1, 1, 1 } }); topology topology; topology.add(input_layout("input", input->get_layout())); - topology.add(one_hot("output", "input", tensor(1, 1, 1, 50), 5)); + topology.add(one_hot("output", "input", tensor(1, 1, 1, 50), 5, 2)); std::string msg_to_find = "Incorrect parameters configuration: one_hot_axis should be less or equal to 4."; EXPECT_ANY_THROW(check_exception_massage(engine, topology, msg_to_find)); @@ -825,11 +823,11 @@ TEST(one_hot_error, basic_error_wrong_axis) { TEST(one_hot_error, basic_error_bad_shape) { auto& engine = get_test_engine(); - auto input = engine.allocate_memory({ data_types::i32, format::bfyx,{ 1, 1, 1, 1 } }); + auto input = engine.allocate_memory({ data_types::i32, format::bfyx, tensor{ 1, 1, 1, 1 } }); topology topology; topology.add(input_layout("input", input->get_layout())); - topology.add(one_hot("output", "input", tensor(1, 5, 1, 50), 2)); + topology.add(one_hot("output", "input", tensor(1, 5, 1, 50), 2, 2)); std::string msg_to_find = "Incorrect parameters configuration: shape does not fit input size."; EXPECT_ANY_THROW(check_exception_massage(engine, topology, msg_to_find)); From dc83410cd7cb91fe419fd1cd7376888d9437d3fa Mon Sep 17 00:00:00 2001 From: Alina Kladieva Date: Thu, 31 Mar 2022 17:38:03 +0300 Subject: [PATCH 35/39] Revert "Skip sporadic GPU canInferOnUserQueue test case (#11310)" (#11362) This reverts commit 458378e9e7a9ce4e2e1e5dd85f370e8bfcb7ad40. --- .../plugin/gpu/shared_tests_instances/skip_tests_config.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tests/functional/plugin/gpu/shared_tests_instances/skip_tests_config.cpp b/src/tests/functional/plugin/gpu/shared_tests_instances/skip_tests_config.cpp index 515ef6db641..682f065e616 100644 --- a/src/tests/functional/plugin/gpu/shared_tests_instances/skip_tests_config.cpp +++ b/src/tests/functional/plugin/gpu/shared_tests_instances/skip_tests_config.cpp @@ -99,7 +99,5 @@ std::vector disabledTestPatterns() { R"(.*registerPluginsXMLUnicodePath.*)", // Not supported yet R"(.*CompileModelCacheTestBase.*)", - // Issue: 83014 - R"(.*smoke_RemoteBlob.*canInferOnUserQueue.*)", }; } From d99104cf5527393f388d7afe4910d8243ece6fb9 Mon Sep 17 00:00:00 2001 From: Ekaterina Aidova Date: Thu, 31 Mar 2022 17:53:41 +0300 Subject: [PATCH 36/39] [OMZ]: update submodule (#11305) --- thirdparty/open_model_zoo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thirdparty/open_model_zoo b/thirdparty/open_model_zoo index 46db87e2c7b..3fb7f248f3d 160000 --- a/thirdparty/open_model_zoo +++ b/thirdparty/open_model_zoo @@ -1 +1 @@ -Subproject commit 46db87e2c7b1324ffccc472fc405582df220e032 +Subproject commit 3fb7f248f3d86a0e223a5621a1208d93fa8510c8 From 3c724a1deeece56ddcf41c9bdbaa5d9068277789 Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Thu, 31 Mar 2022 18:05:59 +0300 Subject: [PATCH 37/39] Build with system TBB (#11244) * Build with system TBB * Fixes * Check whether system TBB is available * Try to fix ONNX Runtime build with system TBB * Test * Fixed compilation of threading.cpp * Fixed unset of cache dirs * Limit dearch paths of TBB * Try to enable pip packages with custom TBB * Fix for TBB 2021.2 * Install only needed TBB libraries * Install TBB from system to pip package * Reverted usage of TBBROOT * Fixed oneTBB case * Try to fix Android * Escape some paths * Added samples path * Fixed TBBBind usage for case of system TBB --- .ci/azure/linux.yml | 1 + CMakeLists.txt | 4 +- cmake/dependencies.cmake | 73 +++++++----- cmake/developer_package/TBBConfig.cmake | 5 +- cmake/developer_package/features.cmake | 6 +- cmake/developer_package/linux_name.cmake | 4 + cmake/features.cmake | 27 ++++- cmake/templates/OpenVINOConfig.cmake.in | 32 +++-- install_build_dependencies.sh | 1 + .../c/common/opencv_c_wrapper/CMakeLists.txt | 4 +- samples/cpp/benchmark_app/README.md | 4 - src/cmake/ie_parallel.cmake | 91 ++++++--------- src/cmake/install_tbb.cmake | 110 ++++++++++++++---- src/core/tests/CMakeLists.txt | 2 +- src/inference/CMakeLists.txt | 33 +++--- src/plugins/intel_cpu/CMakeLists.txt | 20 ++-- .../intel_cpu/thirdparty/CMakeLists.txt | 3 +- src/plugins/intel_gna/CMakeLists.txt | 2 +- .../inference_engine/CMakeLists.txt | 8 +- src/tests_deprecated/helpers/CMakeLists.txt | 2 +- 20 files changed, 275 insertions(+), 157 deletions(-) diff --git a/.ci/azure/linux.yml b/.ci/azure/linux.yml index 7cccd692c8f..3c580b98887 100644 --- a/.ci/azure/linux.yml +++ b/.ci/azure/linux.yml @@ -350,6 +350,7 @@ jobs: - script: | export PATH=$HOME/.local/bin:$PATH export IE_APP_PATH=$(INSTALL_DIR)/samples_bin + export LD_LIBRARY_PATH=$IE_APP_PATH:$LD_LIBRARY_PATH export IE_APP_PYTHON_PATH=$(INSTALL_DIR)/samples/python/ export SHARE=$(INSTALL_DIR)/tests/smoke_tests/samples_smoke_tests_data/ export WORKSPACE=$(INSTALL_DIR) diff --git a/CMakeLists.txt b/CMakeLists.txt index fb9dc63925e..beb02142840 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ if(DEFINED BUILD_SHARED_LIBS AND NOT BUILD_SHARED_LIBS) # 'target_link_libraries' does not work correctly when called from - # different directly where 'add_library' is called: CMake generates + # different directory where 'add_library' is called: CMake generates # incorrect OpenVINOConfig.cmake in this case cmake_minimum_required(VERSION 3.17) else() @@ -17,8 +17,6 @@ if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "Release" CACHE STRING "CMake build type" FORCE) endif() -set(IE_MAIN_SOURCE_DIR ${OpenVINO_SOURCE_DIR}/inference-engine) - find_package(IEDevScripts REQUIRED PATHS "${OpenVINO_SOURCE_DIR}/cmake/developer_package" NO_CMAKE_FIND_ROOT_PATH diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index f2636136e26..b00d7e07e2c 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -23,9 +23,7 @@ message(STATUS "MODELS_PATH=" ${MODELS_PATH}) fetch_models_and_validation_set() -if(COMMAND get_linux_name) - get_linux_name(LINUX_OS_NAME) -endif() +get_linux_name(LINUX_OS_NAME) if(CMAKE_CROSSCOMPILING AND CMAKE_HOST_SYSTEM_NAME MATCHES Linux AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*") set(protoc_version "3.18.2") @@ -93,7 +91,7 @@ if(THREADING STREQUAL "OMP") endif() ## TBB package -if(THREADING STREQUAL "TBB" OR THREADING STREQUAL "TBB_AUTO") +if(THREADING STREQUAL "TBB" OR THREADING STREQUAL "TBB_AUTO" AND NOT ENABLE_SYSTEM_TBB) reset_deps_cache(TBBROOT TBB_DIR) if(DEFINED ENV{THIRDPARTY_SERVER_PATH}) @@ -109,16 +107,6 @@ if(THREADING STREQUAL "TBB" OR THREADING STREQUAL "TBB_AUTO") TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" SHA256 "f1c9b9e2861efdaa01552bd25312ccbc5feeb45551e5f91ae61e29221c5c1479") - if(ENABLE_TBBBIND_2_5) - RESOLVE_DEPENDENCY(TBBBIND_2_5 - ARCHIVE_WIN "tbbbind_2_5_static_win_v1.zip" - TARGET_PATH "${TEMP}/tbbbind_2_5" - ENVIRONMENT "TBBBIND_2_5_ROOT" - SHA256 "a67afeea8cf194f97968c800dab5b5459972908295242e282045d6b8953573c1") - else() - message(WARNING "prebuilt TBBBIND_2_5 is not available. - Build oneTBB from sources and set TBBROOT environment var before OpenVINO cmake configure") - endif() elseif(ANDROID) # Should be before LINUX due LINUX is detected as well RESOLVE_DEPENDENCY(TBB ARCHIVE_ANDROID "tbb2020_20200404_android.tgz" @@ -131,16 +119,6 @@ if(THREADING STREQUAL "TBB" OR THREADING STREQUAL "TBB_AUTO") TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" SHA256 "95b2f3b0b70c7376a0c7de351a355c2c514b42c4966e77e3e34271a599501008") - if(ENABLE_TBBBIND_2_5) - RESOLVE_DEPENDENCY(TBBBIND_2_5 - ARCHIVE_LIN "tbbbind_2_5_static_lin_v2.tgz" - TARGET_PATH "${TEMP}/tbbbind_2_5" - ENVIRONMENT "TBBBIND_2_5_ROOT" - SHA256 "865e7894c58402233caf0d1b288056e0e6ab2bf7c9d00c9dc60561c484bc90f4") - else() - message(WARNING "prebuilt TBBBIND_2_5 is not available. - Build oneTBB from sources and set TBBROOT environment var before OpenVINO cmake configure") - endif() elseif(LINUX AND AARCH64) RESOLVE_DEPENDENCY(TBB ARCHIVE_LIN "keembay/tbb2020_38404_kmb_lic.tgz" @@ -160,13 +138,54 @@ if(THREADING STREQUAL "TBB" OR THREADING STREQUAL "TBB_AUTO") update_deps_cache(TBBROOT "${TBB}" "Path to TBB root folder") if(EXISTS "${TBBROOT}/lib/cmake/TBB/TBBConfig.cmake") # oneTBB case - update_deps_cache(TBB_DIR "${TBB}/lib/cmake/TBB" "Path to TBB cmake folder") + update_deps_cache(TBB_DIR "${TBBROOT}/lib/cmake/TBB" "Path to TBB cmake folder") + elseif(EXISTS "${TBBROOT}/lib64/cmake/TBB/TBBConfig.cmake") + # 64-bits oneTBB case + update_deps_cache(TBB_DIR "${TBBROOT}/lib64/cmake/TBB" "Path to TBB cmake folder") + elseif(EXISTS "${TBBROOT}/cmake/TBBConfig.cmake") + # custom downloaded or user provided TBB + update_deps_cache(TBB_DIR "${TBBROOT}/cmake" "Path to TBB cmake folder") else() - update_deps_cache(TBB_DIR "${TBB}/cmake" "Path to TBB cmake folder") + message(FATAL_ERROR "Failed to find TBBConfig.cmake in ${TBBROOT} tree") endif() update_deps_cache(TBBBIND_2_5_DIR "${TBBBIND_2_5}/cmake" "Path to TBBBIND_2_5 cmake folder") + debug_message(STATUS "tbb=" ${TBB}) + debug_message(STATUS "tbb_dir=" ${TBB_DIR}) + debug_message(STATUS "tbbroot=" ${TBBROOT}) + + if(DEFINED IE_PATH_TO_DEPS) + unset(IE_PATH_TO_DEPS) + endif() +endif() + +## TBBBind_2_5 package +if(ENABLE_TBBBIND_2_5) + if(DEFINED ENV{THIRDPARTY_SERVER_PATH}) + set(IE_PATH_TO_DEPS "$ENV{THIRDPARTY_SERVER_PATH}") + elseif(DEFINED THIRDPARTY_SERVER_PATH) + set(IE_PATH_TO_DEPS "${THIRDPARTY_SERVER_PATH}") + endif() + + if(WIN32 AND X86_64) + RESOLVE_DEPENDENCY(TBBBIND_2_5 + ARCHIVE_WIN "tbbbind_2_5_static_win_v1.zip" + TARGET_PATH "${TEMP}/tbbbind_2_5" + ENVIRONMENT "TBBBIND_2_5_ROOT" + SHA256 "a67afeea8cf194f97968c800dab5b5459972908295242e282045d6b8953573c1") + elseif(LINUX AND X86_64) + RESOLVE_DEPENDENCY(TBBBIND_2_5 + ARCHIVE_LIN "tbbbind_2_5_static_lin_v2.tgz" + TARGET_PATH "${TEMP}/tbbbind_2_5" + ENVIRONMENT "TBBBIND_2_5_ROOT" + SHA256 "865e7894c58402233caf0d1b288056e0e6ab2bf7c9d00c9dc60561c484bc90f4") + else() + message(WARNING "prebuilt TBBBIND_2_5 is not available. +Build oneTBB from sources and set TBBROOT environment var before OpenVINO cmake configure") + endif() + + update_deps_cache(TBBBIND_2_5_DIR "${TBBBIND_2_5}/cmake" "Path to TBBBIND_2_5 cmake folder") if(DEFINED IE_PATH_TO_DEPS) unset(IE_PATH_TO_DEPS) @@ -265,8 +284,6 @@ else() reset_deps_cache(OpenCV_DIR) endif() -include(${OpenVINO_SOURCE_DIR}/src/cmake/ie_parallel.cmake) - if(ENABLE_INTEL_GNA) reset_deps_cache( GNA_EXT_DIR diff --git a/cmake/developer_package/TBBConfig.cmake b/cmake/developer_package/TBBConfig.cmake index 5df466eca6b..908125a8faf 100644 --- a/cmake/developer_package/TBBConfig.cmake +++ b/cmake/developer_package/TBBConfig.cmake @@ -26,10 +26,7 @@ endif() find_package(TBB CONFIG PATHS ${TBBROOT}/cmake - ${TBBROOT}/lib/cmake/TBB # oneTBB case ${IEDevScripts_DIR}/${IE_OWN_TBB_CONFIG} - NO_CMAKE_FIND_ROOT_PATH - NO_DEFAULT_PATH -) + NO_DEFAULT_PATH) find_package_handle_standard_args(TBB CONFIG_MODE) diff --git a/cmake/developer_package/features.cmake b/cmake/developer_package/features.cmake index a60802940ee..92096d49bd8 100644 --- a/cmake/developer_package/features.cmake +++ b/cmake/developer_package/features.cmake @@ -7,7 +7,7 @@ include(target_flags) # FIXME: there are compiler failures with LTO and Cross-Compile toolchains. Disabling for now, but # this must be addressed in a proper way -ie_dependent_option (ENABLE_LTO "Enable Link Time Optimization" OFF "LINUX;NOT CMAKE_CROSSCOMPILING; CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.9" OFF) +ie_dependent_option (ENABLE_LTO "Enable Link Time Optimization" OFF "LINUX;NOT CMAKE_CROSSCOMPILING;CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.9" OFF) ie_option (OS_FOLDER "create OS dedicated folder in output" OFF) @@ -79,6 +79,4 @@ if(ENABLE_AVX512F) endif() endif() -if (VERBOSE_BUILD) - set(CMAKE_VERBOSE_MAKEFILE ON CACHE BOOL "" FORCE) -endif() +set(CMAKE_VERBOSE_MAKEFILE ${VERBOSE_BUILD} CACHE BOOL "" FORCE) diff --git a/cmake/developer_package/linux_name.cmake b/cmake/developer_package/linux_name.cmake index 904cc63236a..d3a4f949395 100644 --- a/cmake/developer_package/linux_name.cmake +++ b/cmake/developer_package/linux_name.cmake @@ -31,4 +31,8 @@ if (LINUX) set(${res_var} NOTFOUND PARENT_SCOPE) endif () endfunction() +else() + function(get_linux_name res_var) + set(${res_var} NOTFOUND PARENT_SCOPE) + endfunction() endif () diff --git a/cmake/features.cmake b/cmake/features.cmake index a68bbdb65b9..d122a99e22d 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -82,7 +82,7 @@ else() set(ENABLE_TBBBIND_2_5_DEFAULT OFF) endif() -ie_dependent_option (ENABLE_TBBBIND_2_5 "Enable TBBBind_2_5 static usage in OpenVINO runtime" ON "ENABLE_TBBBIND_2_5_DEFAULT" OFF) +ie_dependent_option (ENABLE_TBBBIND_2_5 "Enable TBBBind_2_5 static usage in OpenVINO runtime" ${ENABLE_TBBBIND_2_5_DEFAULT} "THREADING MATCHES TBB" OFF) ie_dependent_option (ENABLE_INTEL_GNA "GNA support for inference engine" ON "NOT APPLE;NOT ANDROID;X86_64;CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 5.4" OFF) @@ -136,6 +136,31 @@ ie_dependent_option(ENABLE_TBB_RELEASE_ONLY "Only Release TBB libraries are link ie_dependent_option (ENABLE_SYSTEM_PUGIXML "use the system copy of pugixml" OFF "BUILD_SHARED_LIBS" OFF) +get_linux_name(LINUX_OS_NAME) +if(LINUX_OS_NAME MATCHES "^Ubuntu [0-9]+\.[0-9]+$" AND NOT DEFINED ENV{TBBROOT}) + # Debian packages are enabled on Ubuntu systems + set(ENABLE_SYSTEM_TBB_DEFAULT ON) + + # check whether "default" (customly provided or system one) is available + find_package(TBB QUIET) + if(NOT TBB_FOUND) + message(WARNING "System TBB is not found, custom TBB version will be downloaded from shared drive") + # we still need to download prebuilt version of TBB + set(ENABLE_SYSTEM_TBB_DEFAULT OFF) + endif() + + # remove found TBB (or invalid TBB_DIR-NOTFOUND) from cache + unset(TBB_DIR CACHE) + unset(TBB_DIR) + unset(TBB_FOUND) + unset(TBB_IMPORTED_TARGETS) + unset(TBB_VERSION) +else() + set(ENABLE_SYSTEM_TBB_DEFAULT OFF) +endif() + +ie_dependent_option (ENABLE_SYSTEM_TBB "use the system version of TBB" ${ENABLE_SYSTEM_TBB_DEFAULT} "THREADING MATCHES TBB;LINUX" OFF) + ie_option (ENABLE_DEBUG_CAPS "enable OpenVINO debug capabilities at runtime" OFF) ie_dependent_option (ENABLE_GPU_DEBUG_CAPS "enable GPU debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS" OFF) diff --git a/cmake/templates/OpenVINOConfig.cmake.in b/cmake/templates/OpenVINOConfig.cmake.in index 2c755836a77..ddd051a20d8 100644 --- a/cmake/templates/OpenVINOConfig.cmake.in +++ b/cmake/templates/OpenVINOConfig.cmake.in @@ -12,7 +12,7 @@ # * `Runtime`: OpenVINO C++ and C Core & Inference Runtime, frontend common # * `ONNX`: OpenVINO ONNX frontend # * `Paddle`: OpenVINO Paddle frontend -# * `TF`: OpenVINO TensorFlow frontend +# * `TensorFlow`: OpenVINO TensorFlow frontend # # If no components are specified, `Runtime` component is provided: # @@ -147,14 +147,29 @@ set(_ov_package_prefix_dir "${PACKAGE_PREFIX_DIR}") set(THREADING "@THREADING@") if((THREADING STREQUAL "TBB" OR THREADING STREQUAL "TBB_AUTO") AND NOT TBB_FOUND) - set_and_check(_tbb_dir "@PACKAGE_IE_TBB_DIR@") + set(enable_system_tbb "@ENABLE_SYSTEM_TBB@") + if(NOT enable_system_tbb) + set_and_check(_tbb_dir "@PACKAGE_IE_TBB_DIR@") + set(find_package_tbb_extra_args + CONFIG + PATHS + # oneTBB case exposed via export TBBROOT= + "$ENV{TBBROOT}/lib64/cmake/TBB" + "$ENV{TBBROOT}/lib/cmake/TBB" + # "$ENV{TBB_DIR}" + # for custom TBB exposed via cmake -DTBBROOT= + "${TBBROOT}/cmake" + # _tbb_dir points to TBB_DIR (custom | temp | system) used to build OpenVINO + ${_tbb_dir} + NO_CMAKE_FIND_ROOT_PATH + NO_DEFAULT_PATH) + unset(_tbb_dir) + endif() + unset(enable_system_tbb) + _ov_find_dependency(TBB COMPONENTS tbb tbbmalloc - CONFIG - PATHS ${TBBROOT}/cmake - ${_tbb_dir} - NO_CMAKE_FIND_ROOT_PATH - NO_DEFAULT_PATH) + ${find_package_tbb_extra_args}) set(install_tbbbind "@install_tbbbind@") if(install_tbbbind) @@ -165,6 +180,7 @@ if((THREADING STREQUAL "TBB" OR THREADING STREQUAL "TBB_AUTO") AND NOT TBB_FOUND NO_DEFAULT_PATH) set_target_properties(${TBBBIND_2_5_IMPORTED_TARGETS} PROPERTIES IMPORTED_GLOBAL ON) endif() + unset(install_tbbbind) endif() _ov_find_dependency(Threads) @@ -176,7 +192,7 @@ if(ENABLE_INTEL_GNA AND NOT ENABLE_INTEL_GNA_SHARED AND NOT libGNA_FOUND) _ov_find_dependency(libGNA COMPONENTS KERNEL CONFIG - PATHS ${CMAKE_CURRENT_LIST_DIR} + PATHS "${CMAKE_CURRENT_LIST_DIR}" NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH) endif() diff --git a/install_build_dependencies.sh b/install_build_dependencies.sh index 527f56980aa..8a7f11c6700 100755 --- a/install_build_dependencies.sh +++ b/install_build_dependencies.sh @@ -64,6 +64,7 @@ if [ -f /etc/lsb-release ]; then libgstreamer1.0-0 \ gstreamer1.0-plugins-base \ libusb-1.0-0-dev \ + libtbb-dev \ libtinfo5 \ libopenblas-dev if apt-cache search --names-only '^libjson-c2'| grep -q libjson-c2; then diff --git a/samples/c/common/opencv_c_wrapper/CMakeLists.txt b/samples/c/common/opencv_c_wrapper/CMakeLists.txt index a8e6e51721d..2755579a0fe 100644 --- a/samples/c/common/opencv_c_wrapper/CMakeLists.txt +++ b/samples/c/common/opencv_c_wrapper/CMakeLists.txt @@ -16,10 +16,10 @@ find_package(OpenCV COMPONENTS core imgproc imgcodecs QUIET) if(NOT OpenCV_FOUND) message(WARNING "OPENCV is disabled or not found, ${TARGET_NAME} is built without OPENCV support") else() - add_definitions(-DUSE_OPENCV) + target_compile_definitions(${TARGET_NAME} PRIVATE USE_OPENCV) endif() -target_link_libraries(${TARGET_NAME} PUBLIC ${OpenCV_LIBRARIES}) +target_link_libraries(${TARGET_NAME} PRIVATE ${OpenCV_LIBRARIES}) target_include_directories(${TARGET_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}") diff --git a/samples/cpp/benchmark_app/README.md b/samples/cpp/benchmark_app/README.md index 1e9bc3345de..9935f322941 100644 --- a/samples/cpp/benchmark_app/README.md +++ b/samples/cpp/benchmark_app/README.md @@ -167,12 +167,8 @@ This section provides step-by-step instructions on how to run the Benchmark Tool ```sh omz_downloader --name googlenet-v1 -o ``` -<<<<<<< HEAD 3. Convert the model to the OpenVINO IR format. Run the Model Optimizer using the `mo` command with the path to the model, model format and output directory to generate the IR files: -======= -2. Convert the model to the OpenVINO IR format. Run the Model Optimizer using the `mo` command with the path to the model, model format (which must be FP32 for CPU and FPG) and output directory to generate the IR files: ->>>>>>> cf8ccb590a... Removed obsolete code snippets (#11061) ```sh mo --input_model /public/googlenet-v1/googlenet-v1.caffemodel --data_type FP32 --output_dir ``` diff --git a/src/cmake/ie_parallel.cmake b/src/cmake/ie_parallel.cmake index a036b019c84..b52d593a9ea 100644 --- a/src/cmake/ie_parallel.cmake +++ b/src/cmake/ie_parallel.cmake @@ -2,30 +2,26 @@ # SPDX-License-Identifier: Apache-2.0 # -function(set_ie_threading_interface_for TARGET_NAME) - macro(ext_message TRACE_LEVEL) - if (TRACE_LEVEL STREQUAL FATAL_ERROR) - if(InferenceEngine_FIND_REQUIRED) - message(FATAL_ERROR "${ARGN}") - elseif(NOT InferenceEngine_FIND_QUIETLY) - message(WARNING "${ARGN}") - endif() - return() - elseif(NOT InferenceEngine_FIND_QUIETLY) - message(${TRACE_LEVEL} "${ARGN}") - endif () - endmacro() +macro(ov_find_tbb) + if(THREADING STREQUAL "TBB" OR THREADING STREQUAL "TBB_AUTO" AND NOT TBB_FOUND) + find_package(TBB COMPONENTS tbb tbbmalloc) - if (THREADING STREQUAL "TBB" OR THREADING STREQUAL "TBB_AUTO" AND NOT TBB_FOUND) - if(IEDevScripts_DIR) + # try to find TBB via custom scripts if have not found by default + if(NOT TBB_FOUND AND IEDevScripts_DIR) + # remove invalid TBB_DIR=TBB_DIR-NOTFOUND from cache + unset(TBB_DIR CACHE) + unset(TBB_DIR) + + # use our custom scripts for old TBB versions + # which are exposed via `export TBBROOT=` + # see https://github.com/openvinotoolkit/openvino/pull/1288 find_package(TBB COMPONENTS tbb tbbmalloc - PATHS IEDevScripts_DIR + PATHS ${IEDevScripts_DIR} NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH) - else() - find_dependency(TBB COMPONENTS tbb tbbmalloc) endif() - # oneTBB does not define TBB_IMPORTED_TARGETS + + # WA for oneTBB: it does not define TBB_IMPORTED_TARGETS if(TBB_FOUND AND NOT TBB_IMPORTED_TARGETS) foreach(target TBB::tbb TBB::tbbmalloc) if(TARGET ${target}) @@ -33,15 +29,23 @@ function(set_ie_threading_interface_for TARGET_NAME) endif() endforeach() endif() + + # set variables to parent scope to prevent multiple invocations of find_package(TBB) + # at the same CMakeLists.txt; invocations in different directories are allowed set(TBB_FOUND ${TBB_FOUND} PARENT_SCOPE) set(TBB_IMPORTED_TARGETS ${TBB_IMPORTED_TARGETS} PARENT_SCOPE) set(TBB_VERSION ${TBB_VERSION} PARENT_SCOPE) if (NOT TBB_FOUND) set(THREADING "SEQ" PARENT_SCOPE) - ext_message(WARNING "TBB was not found by the configured TBB_DIR/TBBROOT path.\ - SEQ method will be used.") + message(WARNING "TBB was not found by the configured TBB_DIR/TBBROOT path.\ + SEQ method will be used.") endif () endif() +endmacro() + +function(set_ie_threading_interface_for TARGET_NAME) + # find TBB + ov_find_tbb() get_target_property(target_type ${TARGET_NAME} TYPE) @@ -59,32 +63,11 @@ function(set_ie_threading_interface_for TARGET_NAME) # TODO: why TBB propogates its headers to inference_engine? set(LINK_TYPE "PRIVATE") else() - ext_message(WARNING "Unknown target type") + message(WARNING "Unknown target type") endif() function(ie_target_link_libraries TARGET_NAME LINK_TYPE) - if(CMAKE_VERSION VERSION_LESS "3.12.0") - if(NOT target_type STREQUAL "OBJECT_LIBRARY") - target_link_libraries(${TARGET_NAME} ${LINK_TYPE} ${ARGN}) - else() - # Object library may not link to anything. - # To add interface include definitions and compile options explicitly. - foreach(ITEM IN LISTS ARGN) - if(TARGET ${ITEM}) - get_target_property(compile_options ${ITEM} INTERFACE_COMPILE_OPTIONS) - if (compile_options) - target_compile_options(${TARGET_NAME} ${LINK_TYPE} ${compile_options}) - endif() - get_target_property(compile_definitions ${ITEM} INTERFACE_COMPILE_DEFINITIONS) - if (compile_definitions) - target_compile_definitions(${TARGET_NAME} ${LINK_TYPE} ${compile_definitions}) - endif() - endif() - endforeach() - endif() - else() - target_link_libraries(${TARGET_NAME} ${LINK_TYPE} ${ARGN}) - endif() + target_link_libraries(${TARGET_NAME} ${LINK_TYPE} ${ARGN}) # include directories as SYSTEM foreach(library IN LISTS ARGN) @@ -92,8 +75,11 @@ function(set_ie_threading_interface_for TARGET_NAME) get_target_property(include_directories ${library} INTERFACE_INCLUDE_DIRECTORIES) if(include_directories) foreach(include_directory IN LISTS include_directories) - target_include_directories(${TARGET_NAME} SYSTEM BEFORE - ${LINK_TYPE} $) + # cannot include /usr/include headers as SYSTEM + if(NOT "${include_directory}" MATCHES "^/usr.*$") + target_include_directories(${TARGET_NAME} SYSTEM BEFORE + ${LINK_TYPE} $) + endif() endforeach() endif() endif() @@ -108,8 +94,8 @@ function(set_ie_threading_interface_for TARGET_NAME) ie_target_link_libraries(${TARGET_NAME} ${LINK_TYPE} ${TBB_IMPORTED_TARGETS}) else () set(THREADING "SEQ" PARENT_SCOPE) - ext_message(WARNING "TBB was not found by the configured TBB_DIR path.\ - SEQ method will be used for ${TARGET_NAME}") + message(WARNING "TBB was not found by the configured TBB_DIR path.\ + SEQ method will be used for ${TARGET_NAME}") endif () elseif (THREADING STREQUAL "OMP") if (WIN32) @@ -133,19 +119,19 @@ function(set_ie_threading_interface_for TARGET_NAME) if (NOT OMP_LIBRARIES_RELEASE) find_library(OMP_LIBRARIES_RELEASE ${omp_lib_name} ${lib_rel_path} NO_DEFAULT_PATH) - ext_message(STATUS "OMP Release lib: ${OMP_LIBRARIES_RELEASE}") + message(STATUS "OMP Release lib: ${OMP_LIBRARIES_RELEASE}") if (NOT LINUX) find_library(OMP_LIBRARIES_DEBUG ${omp_lib_name} ${lib_dbg_path} NO_DEFAULT_PATH) if (OMP_LIBRARIES_DEBUG) - ext_message(STATUS "OMP Debug lib: ${OMP_LIBRARIES_DEBUG}") + message(STATUS "OMP Debug lib: ${OMP_LIBRARIES_DEBUG}") else () - ext_message(WARNING "OMP Debug binaries are missed.") + message(WARNING "OMP Debug binaries are missed.") endif () endif () endif () if (NOT OMP_LIBRARIES_RELEASE) - ext_message(WARNING "Intel OpenMP not found. Intel OpenMP support will be disabled. ${IE_THREAD_DEFINE} is defined") + message(WARNING "Intel OpenMP not found. Intel OpenMP support will be disabled. ${IE_THREAD_DEFINE} is defined") set(THREADING "SEQ" PARENT_SCOPE) else () set(IE_THREAD_DEFINE "IE_THREAD_OMP") @@ -174,7 +160,6 @@ function(set_ie_threading_interface_for TARGET_NAME) ie_target_link_libraries(${TARGET_NAME} ${LINK_TYPE} ${OMP_LIBRARIES_RELEASE}) endif () endif () - endif () target_compile_definitions(${TARGET_NAME} ${LINK_TYPE} -DIE_THREAD=${IE_THREAD_DEFINE}) diff --git a/src/cmake/install_tbb.cmake b/src/cmake/install_tbb.cmake index 6e1ab3f3982..7198b41da69 100644 --- a/src/cmake/install_tbb.cmake +++ b/src/cmake/install_tbb.cmake @@ -2,6 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 # +include(cmake/ie_parallel.cmake) + +# pre-find TBB: need to provide TBB_IMPORTED_TARGETS used for installation +ov_find_tbb() + if(ENABLE_TBBBIND_2_5) # try to find prebuilt version of tbbbind_2_5 find_package(TBBBIND_2_5 QUIET) @@ -12,6 +17,8 @@ if(ENABLE_TBBBIND_2_5) if(NOT BUILD_SHARED_LIBS) set(install_tbbbind ON) endif() + else() + message(WARNING "Static tbbbind_2_5 package is not found") endif() endif() @@ -28,34 +35,93 @@ if(install_tbbbind) list(APPEND PATH_VARS "IE_TBBBIND_DIR") endif() -# install only downloaded TBB, system one is not installed -if(THREADING MATCHES "^(TBB|TBB_AUTO)$" AND TBBROOT MATCHES ${TEMP}) +# install only downloaded | custom TBB, system one is not installed +# - downloaded TBB should be a part of all packages +# - custom TBB provided by users, needs to be a part of wheel packages +# - TODO: system TBB also needs to be a part of wheel packages +if(THREADING MATCHES "^(TBB|TBB_AUTO)$" AND + (TBB MATCHES ${TEMP} OR DEFINED ENV{TBBROOT} OR ENABLE_SYSTEM_TBB)) ie_cpack_add_component(tbb REQUIRED) - ie_cpack_add_component(tbb_dev REQUIRED) list(APPEND core_components tbb) - list(APPEND core_dev_components tbb_dev) - install(DIRECTORY "${TBB}/lib" - DESTINATION runtime/3rdparty/tbb - COMPONENT tbb) - # Windows only - if(EXISTS "${TBB}/bin") - install(DIRECTORY "${TBB}/bin" + if(TBB MATCHES ${TEMP}) + set(tbb_downloaded ON) + elseif(DEFINED ENV{TBB}) + set(tbb_custom ON) + endif() + + if(tbb_custom OR ENABLE_SYSTEM_TBB) + # since the setup.py for pip installs tbb component + # explicitly, it's OK to put EXCLUDE_FROM_ALL to such component + # to ignore from IRC distribution + set(exclude_from_all EXCLUDE_FROM_ALL) + endif() + + if(ENABLE_SYSTEM_TBB) + # need to take locations of actual libraries and install them + foreach(tbb_lib IN LISTS TBB_IMPORTED_TARGETS) + get_target_property(tbb_loc ${tbb_lib} IMPORTED_LOCATION_RELEASE) + # depending on the TBB, tbb_loc can be in form: + # - libtbb.so.x.y + # - libtbb.so.x + # - libtbb.so + # We need to install such files + get_filename_component(name_we "${tbb_loc}" NAME_WE) + get_filename_component(dir "${tbb_loc}" DIRECTORY) + file(GLOB tbb_files "${dir}/${name_we}.*") + foreach(tbb_file IN LISTS tbb_files) + if(tbb_file MATCHES "^.*\.${CMAKE_SHARED_LIBRARY_SUFFIX}(\.[0-9]+)*$") + install(FILES "${tbb_file}" + DESTINATION runtime/3rdparty/tbb/lib + COMPONENT tbb ${exclude_from_all}) + endif() + endforeach() + endforeach() + else() + if(WIN32) + install(DIRECTORY "${TBB}/bin" + DESTINATION runtime/3rdparty/tbb + COMPONENT tbb ${exclude_from_all}) + elseif(tbb_custom OR tbb_downloaded) + install(DIRECTORY "${TBB}/lib" + DESTINATION runtime/3rdparty/tbb + COMPONENT tbb ${exclude_from_all} + FILES_MATCHING + # install only versioned shared libraries + REGEX "^.*\.${CMAKE_SHARED_LIBRARY_SUFFIX}(\.[0-9]+)*$") + endif() + endif() + + # development files are needed only for 'tbb_downloaded' case + # which we are going to distribute + if(tbb_downloaded) + ie_cpack_add_component(tbb_dev REQUIRED) + list(APPEND core_dev_components tbb_dev) + + install(FILES "${TBB}/LICENSE" DESTINATION runtime/3rdparty/tbb COMPONENT tbb) - endif() - install(FILES "${TBB}/LICENSE" - DESTINATION runtime/3rdparty/tbb - COMPONENT tbb) - set(IE_TBB_DIR_INSTALL "3rdparty/tbb/cmake") - install(FILES "${TBB}/cmake/TBBConfig.cmake" - "${TBB}/cmake/TBBConfigVersion.cmake" - DESTINATION runtime/${IE_TBB_DIR_INSTALL} - COMPONENT tbb_dev) - install(DIRECTORY "${TBB}/include" - DESTINATION runtime/3rdparty/tbb - COMPONENT tbb_dev) + set(IE_TBB_DIR_INSTALL "3rdparty/tbb/cmake") + install(FILES "${TBB}/cmake/TBBConfig.cmake" + "${TBB}/cmake/TBBConfigVersion.cmake" + DESTINATION runtime/${IE_TBB_DIR_INSTALL} + COMPONENT tbb_dev) + install(DIRECTORY "${TBB}/include" + DESTINATION runtime/3rdparty/tbb + COMPONENT tbb_dev) + + if(WIN32) + # .lib files are needed only for Windows + install(DIRECTORY "${TBB}/lib" + DESTINATION runtime/3rdparty/tbb + COMPONENT tbb) + endif() + endif() + + unset(tbb_downloaded) + unset(tbb_custom) + unset(exclude_from_all) endif() # install tbbbind for static OpenVINO case diff --git a/src/core/tests/CMakeLists.txt b/src/core/tests/CMakeLists.txt index 96ac8b23f60..0c857836eaa 100644 --- a/src/core/tests/CMakeLists.txt +++ b/src/core/tests/CMakeLists.txt @@ -421,7 +421,7 @@ set(SRC # For type relaxed types set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/threading.cpp - PROPERTIES INCLUDE_DIRECTORIES $) + PROPERTIES INCLUDE_DIRECTORIES $) if(SUGGEST_OVERRIDE_SUPPORTED) set_source_files_properties(ov_tensor_test.cpp diff --git a/src/inference/CMakeLists.txt b/src/inference/CMakeLists.txt index ddd2fdcbc86..767fbc1b81d 100644 --- a/src/inference/CMakeLists.txt +++ b/src/inference/CMakeLists.txt @@ -123,22 +123,27 @@ ie_faster_build(${TARGET_NAME}_obj UNITY PCH PRIVATE "src/precomp.hpp" ) -target_compile_definitions(${TARGET_NAME}_obj PRIVATE IMPLEMENT_INFERENCE_ENGINE_API - $ - $ - $) +target_compile_definitions(${TARGET_NAME}_obj PRIVATE + IMPLEMENT_INFERENCE_ENGINE_API + $ + $ + $) -target_include_directories(${TARGET_NAME}_obj SYSTEM PRIVATE $ - $ - $ - $) +target_include_directories(${TARGET_NAME}_obj SYSTEM PRIVATE + $ + $ + $ + $) -target_include_directories(${TARGET_NAME}_obj PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src" - "${CMAKE_CURRENT_BINARY_DIR}" # for static ie_plugins.hpp - $<$:$> # for ie_ir_version.hpp - $ - $ - $) +target_include_directories(${TARGET_NAME}_obj PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src" + # for static ie_plugins.hpp + "${CMAKE_CURRENT_BINARY_DIR}" + # for ie_ir_version.hpp + $<$:$> + $ + $ + $) target_link_libraries(${TARGET_NAME}_obj PRIVATE openvino::itt openvino::util ov_core_dev) diff --git a/src/plugins/intel_cpu/CMakeLists.txt b/src/plugins/intel_cpu/CMakeLists.txt index b802cd52729..02fff2aa30d 100644 --- a/src/plugins/intel_cpu/CMakeLists.txt +++ b/src/plugins/intel_cpu/CMakeLists.txt @@ -70,13 +70,16 @@ if(BUILD_SHARED_LIBS) add_library(${TARGET_NAME}_obj OBJECT ${SOURCES} ${HEADERS}) link_system_libraries(${TARGET_NAME}_obj PUBLIC mkldnn) - target_include_directories(${TARGET_NAME}_obj PRIVATE $ - $ - $ - $ - $ - PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src - $) + target_include_directories(${TARGET_NAME}_obj + PRIVATE + $ + $ + $ + $ + $ + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/src + $) target_include_directories(${TARGET_NAME}_obj SYSTEM PUBLIC $) @@ -85,8 +88,7 @@ if(BUILD_SHARED_LIBS) target_compile_definitions(${TARGET_NAME}_obj PRIVATE USE_STATIC_IE IMPLEMENT_INFERENCE_ENGINE_PLUGIN IMPLEMENT_INFERENCE_EXTENSION_API $ - $ - ) + $) set_target_properties(${TARGET_NAME}_obj PROPERTIES EXCLUDE_FROM_ALL ON) diff --git a/src/plugins/intel_cpu/thirdparty/CMakeLists.txt b/src/plugins/intel_cpu/thirdparty/CMakeLists.txt index 651f6738e57..394fb548bd9 100644 --- a/src/plugins/intel_cpu/thirdparty/CMakeLists.txt +++ b/src/plugins/intel_cpu/thirdparty/CMakeLists.txt @@ -23,12 +23,13 @@ function(ie_add_mkldnn) set(DNNL_ENABLE_CONCURRENT_EXEC ON CACHE BOOL "" FORCE) set(DNNL_ENABLE_PRIMITIVE_CACHE OFF CACHE BOOL "" FORCE) ## TODO: try it later set(DNNL_ENABLE_MAX_CPU_ISA ON CACHE BOOL "" FORCE) - set(DNNL_LIBRARY_TYPE STATIC CACHE BOOL "" FORCE) + set(DNNL_LIBRARY_TYPE "STATIC" CACHE STRING "" FORCE) set(DNNL_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) set(DNNL_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(DNNL_CPU_RUNTIME "${THREADING}" CACHE STRING "" FORCE) set(DNNL_GPU_RUNTIME "NONE" CACHE STRING "" FORCE) set(DNNL_BLAS_VENDOR "NONE" CACHE STRING "" FORCE) + set(DNNL_VERBOSE "OFF" CACHE STRING "" FORCE) set(SDL_cmake_included ON) ## to skip internal SDL flags. SDL flags are already set on IE level if (ANDROID OR ((CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") AND NOT (THREADING STREQUAL "OMP"))) set(OpenMP_cmake_included ON) ## to skip "omp simd" inside a code. Lead to some crashes inside NDK LLVM.. diff --git a/src/plugins/intel_gna/CMakeLists.txt b/src/plugins/intel_gna/CMakeLists.txt index 1641aec0183..310ac7581db 100644 --- a/src/plugins/intel_gna/CMakeLists.txt +++ b/src/plugins/intel_gna/CMakeLists.txt @@ -79,7 +79,7 @@ foreach(gna_lib IN LISTS gna_libraries) # and don't install global symlink libgna.so. Exceptions: # 1. On Windows we don't have libraries versioning and need to install all .dlls # 2. For case of static libraries we also need to install .so since it's required for linkage with final application - if(gna_lib MATCHES "^.*${CMAKE_SHARED_LIBRARY_SUFFIX}(\.[0-9]+)+$" OR + if(gna_lib MATCHES "^.*\.${CMAKE_SHARED_LIBRARY_SUFFIX}(\.[0-9]+)+$" OR WIN32 OR NOT BUILD_SHARED_LIBS) install(FILES ${gna_lib} DESTINATION ${IE_CPACK_RUNTIME_PATH} diff --git a/src/tests/functional/inference_engine/CMakeLists.txt b/src/tests/functional/inference_engine/CMakeLists.txt index f9d0bd5df51..ef19096d7c8 100644 --- a/src/tests/functional/inference_engine/CMakeLists.txt +++ b/src/tests/functional/inference_engine/CMakeLists.txt @@ -282,8 +282,14 @@ ie_headers_compilation_with_custom_flags(TEST_SUFFIX PluginApiCxx17 CXX_STANDARD 17 PLUGIN_API) if(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 8.0) + if(TBB_VERSION VERSION_EQUAL 2021.2) + # TBB 2021.2 has a bug in headers with CXX_STANDARD 20 + # see https://github.com/oneapi-src/oneTBB/issues/379 + set(skip_headers_list HEADERS_TO_SKIP "threading/ie_thread_local.hpp") + endif() ie_headers_compilation_with_custom_flags(TEST_SUFFIX PluginApiCxx20 - CXX_STANDARD 20 PLUGIN_API) + ${skip_headers_list} + CXX_STANDARD 20 PLUGIN_API) endif() if(UNIX) diff --git a/src/tests_deprecated/helpers/CMakeLists.txt b/src/tests_deprecated/helpers/CMakeLists.txt index 454d07b7bd3..dbea59733e0 100644 --- a/src/tests_deprecated/helpers/CMakeLists.txt +++ b/src/tests_deprecated/helpers/CMakeLists.txt @@ -13,7 +13,7 @@ file (GLOB HELPERS_INCLUDES ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp ) -## Enable Models multiple search pathes +## Enable Models multiple search paths message("configuring file: ${CMAKE_CURRENT_BINARY_DIR}/test_model_repo.h") function(add_helpers target_name) From 1cb254307ed6730534079543a13fef7fbb4158ee Mon Sep 17 00:00:00 2001 From: Vladimir Paramuzov Date: Thu, 31 Mar 2022 19:46:38 +0300 Subject: [PATCH 38/39] [GPU] Gather params update (#11369) --- .../include/intel_gpu/primitives/gather.hpp | 32 ++- src/plugins/intel_gpu/src/graph/gather.cpp | 12 +- .../intel_gpu/src/graph/impls/ocl/gather.cpp | 61 ++++-- .../intel_gpu/src/plugin/ops/gather.cpp | 53 +---- .../tests/fusions/gather_fusion_test.cpp | 69 +++---- .../tests/test_cases/gather_gpu_test.cpp | 188 +++++++++--------- 6 files changed, 188 insertions(+), 227 deletions(-) diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/gather.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/gather.hpp index 776b994576d..5c2224aa863 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/primitives/gather.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/primitives/gather.hpp @@ -6,6 +6,8 @@ #pragma once #include "primitive.hpp" +#include "openvino/core/shape.hpp" + namespace cldnn { /// @addtogroup cpp_api C++ API /// @{ @@ -19,15 +21,6 @@ namespace cldnn { struct gather : public primitive_base { CLDNN_DECLARE_PRIMITIVE(gather) - enum gather_axis { - along_b, - along_f, - along_x, - along_y, - along_z, - along_w - }; - /// @brief Constructs gather primitive. /// @param id This primitive id. /// @param dict Input dictionary primitive id. @@ -39,23 +32,22 @@ struct gather : public primitive_base { gather(const primitive_id& id, const primitive_id& dict, const primitive_id& idx, - const gather_axis axis, - const format& output_format, - const tensor& output_shape, + const int64_t axis, + const ov::Shape& output_shape, const int64_t batch_dim = 0, const bool support_neg_ind = false, const primitive_id& ext_prim_id = "", - const padding& output_padding = padding() - ) - : primitive_base(id, {dict, idx}, ext_prim_id, output_padding), axis(axis), output_format(output_format), - output_shape(output_shape), batch_dim(batch_dim), support_neg_ind(support_neg_ind) {} + const padding& output_padding = padding()) + : primitive_base(id, {dict, idx}, ext_prim_id, output_padding) + , axis(axis) + , output_shape(output_shape) + , batch_dim(batch_dim) + , support_neg_ind(support_neg_ind) {} /// @brief Gathering axis - gather_axis axis; - /// @brief Gather output format - format output_format; + int64_t axis; /// @brief Gather output shape - tensor output_shape; + ov::Shape output_shape; /// @brief Gathering batch_dim int64_t batch_dim; /// @brief Support negative indexes diff --git a/src/plugins/intel_gpu/src/graph/gather.cpp b/src/plugins/intel_gpu/src/graph/gather.cpp index 3d216a05ac4..31716ce77a2 100644 --- a/src/plugins/intel_gpu/src/graph/gather.cpp +++ b/src/plugins/intel_gpu/src/graph/gather.cpp @@ -19,15 +19,19 @@ layout gather_inst::calc_output_layout(gather_node const& node) { auto desc = node.get_primitive(); auto input_layout = node.input(0).get_output_layout(); - auto output_format = desc->output_format; - auto output_shape = desc->output_shape; + std::vector dims_converted(desc->output_shape.begin(), desc->output_shape.end()); + // extend shape to 4d + for (size_t i = dims_converted.size(); i < 4; i++) { + dims_converted.push_back(1); + } + auto output_format = format::get_default_format(dims_converted.size()); auto output_type = input_layout.data_type; if (node.has_fused_primitives()) { output_type = node.get_fused_output_layout().data_type; } - return layout{output_type, output_format, output_shape}; + return layout{output_type, output_format, tensor(output_format, dims_converted)}; } std::string gather_inst::to_string(gather_node const& node) { @@ -41,7 +45,7 @@ std::string gather_inst::to_string(gather_node const& node) { gather_info.add("input id", input.id()); gather_info.add("axis", desc->axis); gather_info.add("batch_dim", desc->batch_dim); - gather_info.add("output shape", desc->output_shape.to_string()); + gather_info.add("output shape", cldnn::to_string(desc->output_shape)); node_info->add("gather info", gather_info); node_info->dump(primitive_description); diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/gather.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/gather.cpp index 81fcaf63167..11a8832b278 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/gather.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/gather.cpp @@ -14,22 +14,48 @@ using namespace cldnn; namespace cldnn { namespace ocl { -kernel_selector::gather_axis convert_axis(gather::gather_axis axis) { - switch (axis) { - case gather::along_x: - return kernel_selector::gather_axis::X; - case gather::along_y: - return kernel_selector::gather_axis::Y; - case gather::along_z: - return kernel_selector::gather_axis::Z; - case gather::along_w: - return kernel_selector::gather_axis::W; - case gather::along_f: - return kernel_selector::gather_axis::FEATURE; - case gather::along_b: - return kernel_selector::gather_axis::BATCH; - default: - return kernel_selector::gather_axis::X; +static kernel_selector::gather_axis convert_axis(int64_t axis, size_t rank) { + if (axis == 0) { + return kernel_selector::gather_axis::BATCH; + } else if (axis == 1) { + return kernel_selector::gather_axis::FEATURE; + } + + if (rank <= 4) { + switch (axis) { + case 2: return kernel_selector::gather_axis::Y; + case 3: return kernel_selector::gather_axis::X; + case -1: return kernel_selector::gather_axis::Y; + case -2: return kernel_selector::gather_axis::FEATURE; + case -3: return kernel_selector::gather_axis::BATCH; + default: IE_THROW() << "Unsupported gather axis: " << axis; + } + } else if (rank == 5) { + switch (axis) { + case 2: return kernel_selector::gather_axis::Z; + case 3: return kernel_selector::gather_axis::Y; + case 4: return kernel_selector::gather_axis::X; + case -1: return kernel_selector::gather_axis::Y; + case -2: return kernel_selector::gather_axis::Z; + case -3: return kernel_selector::gather_axis::FEATURE; + case -4: return kernel_selector::gather_axis::BATCH; + default: IE_THROW() << "Unsupported gather axis: " << axis; + } + } else if (rank == 6) { + switch (axis) { + case 2: return kernel_selector::gather_axis::W; + case 3: return kernel_selector::gather_axis::Z; + case 4: return kernel_selector::gather_axis::Y; + case 5: return kernel_selector::gather_axis::X; + case -1: return kernel_selector::gather_axis::Y; + case -2: return kernel_selector::gather_axis::Z; + case -3: return kernel_selector::gather_axis::W; + case -4: return kernel_selector::gather_axis::FEATURE; + case -5: return kernel_selector::gather_axis::BATCH; + default: IE_THROW() << "Unsupported gather axis: " << axis; + } + } else { + IE_THROW() << "Unsupported gather axis: " << axis; } } @@ -47,7 +73,8 @@ public: auto gather_optional_params = get_default_optional_params(arg.get_program()); - gather_params.axis = convert_axis(arg.get_primitive()->axis); + auto input_layout = arg.get_dependency(0).get_output_layout(); + gather_params.axis = convert_axis(arg.get_primitive()->axis, input_layout.get_rank()); gather_params.batch_dim = size_t(arg.get_primitive()->batch_dim); gather_params.support_neg_ind = arg.get_primitive()->support_neg_ind; diff --git a/src/plugins/intel_gpu/src/plugin/ops/gather.cpp b/src/plugins/intel_gpu/src/plugin/ops/gather.cpp index bdd8f5d5f3f..3f5ba77a516 100644 --- a/src/plugins/intel_gpu/src/plugin/ops/gather.cpp +++ b/src/plugins/intel_gpu/src/plugin/ops/gather.cpp @@ -14,57 +14,12 @@ namespace ov { namespace runtime { namespace intel_gpu { -static cldnn::gather::gather_axis GetGatherAxis(int32_t axis, cldnn::format inputFormat) { - if (axis == 0) { - return cldnn::gather::gather_axis::along_b; - } else if (axis == 1) { - return cldnn::gather::gather_axis::along_f; - } - - if (inputFormat == cldnn::format::bfyx) { - switch (axis) { - case 2: return cldnn::gather::gather_axis::along_y; - case 3: return cldnn::gather::gather_axis::along_x; - case -1: return cldnn::gather::gather_axis::along_y; - case -2: return cldnn::gather::gather_axis::along_f; - case -3: return cldnn::gather::gather_axis::along_b; - default: IE_THROW() << "Unsupported gather axis: " << axis; - } - } else if (inputFormat == cldnn::format::bfzyx) { - switch (axis) { - case 2: return cldnn::gather::gather_axis::along_z; - case 3: return cldnn::gather::gather_axis::along_y; - case 4: return cldnn::gather::gather_axis::along_x; - case -1: return cldnn::gather::gather_axis::along_y; - case -2: return cldnn::gather::gather_axis::along_z; - case -3: return cldnn::gather::gather_axis::along_f; - case -4: return cldnn::gather::gather_axis::along_b; - default: IE_THROW() << "Unsupported gather axis: " << axis; - } - } else if (inputFormat == cldnn::format::bfwzyx) { - switch (axis) { - case 2: return cldnn::gather::gather_axis::along_w; - case 3: return cldnn::gather::gather_axis::along_z; - case 4: return cldnn::gather::gather_axis::along_y; - case 5: return cldnn::gather::gather_axis::along_x; - case -1: return cldnn::gather::gather_axis::along_y; - case -2: return cldnn::gather::gather_axis::along_z; - case -3: return cldnn::gather::gather_axis::along_w; - case -4: return cldnn::gather::gather_axis::along_f; - case -5: return cldnn::gather::gather_axis::along_b; - default: IE_THROW() << "Unsupported gather axis: " << axis; - } - } else { - IE_THROW() << "Unsupported gather axis: " << axis; - } -} - template void CreateGatherOpBase(Program& p, const std::shared_ptr& op, const int64_t batch_dim = 0, bool support_neg_ind = false) { auto inputPrimitives = p.GetInputPrimitiveIDs(op); std::string layerName = layer_type_name_ID(op); - int32_t axis = static_cast(op->get_axis()); + int64_t axis = op->get_axis(); std::vector reorderedInputs; reorderedInputs.resize(inputPrimitives.size()); @@ -91,13 +46,11 @@ void CreateGatherOpBase(Program& p, const std::shared_ptr& op, const int64_t } } - auto outLayout = DefaultFormatForDims(op->get_output_shape(0).size()); auto gatherPrim = cldnn::gather(layerName, reorderedInputs[0], reorderedInputs[1], - GetGatherAxis(axis, DefaultFormatForDims(op->get_input_shape(0).size())), - outLayout, - tensor_from_dims(op->get_output_shape(0)), + axis, + op->get_output_shape(0), batch_dim, support_neg_ind, op->get_friendly_name()); diff --git a/src/plugins/intel_gpu/tests/fusions/gather_fusion_test.cpp b/src/plugins/intel_gpu/tests/fusions/gather_fusion_test.cpp index 012b7551ca5..7beba20559c 100644 --- a/src/plugins/intel_gpu/tests/fusions/gather_fusion_test.cpp +++ b/src/plugins/intel_gpu/tests/fusions/gather_fusion_test.cpp @@ -20,9 +20,8 @@ namespace { struct gather_test_params { tensor dictionary_shape; tensor indices_shape; - tensor out_shape; - format out_format; - cldnn::gather::gather_axis axis; + ov::Shape out_shape; + int64_t axis; data_types data_type; format input_format; data_types default_type; @@ -52,26 +51,12 @@ public: } size_t get_axis_dim(gather_test_params& p) { - switch (p.axis) { - case cldnn::gather::gather_axis::along_x: - return p.dictionary_shape.spatial[0]; - case cldnn::gather::gather_axis::along_y: - return p.dictionary_shape.spatial[1]; - case cldnn::gather::gather_axis::along_z: - return p.dictionary_shape.spatial[2]; - case cldnn::gather::gather_axis::along_w: - return p.dictionary_shape.spatial[3]; - case cldnn::gather::gather_axis::along_f: - return p.dictionary_shape.feature[0]; - case cldnn::gather::gather_axis::along_b: - return p.dictionary_shape.batch[0]; - default: - return 1; - } + auto in_layout = get_input_layout(p); + return in_layout.get_dims()[p.axis]; } layout get_per_channel_layout(gather_test_params& p) { - return layout{ p.default_type, p.default_format, tensor{ 1, p.out_shape.feature[0], 1, 1 } }; + return layout{ p.default_type, p.default_format, tensor{ 1, static_cast(p.out_shape[1]), 1, 1 } }; } }; } // namespace @@ -80,29 +65,29 @@ public: /* ------------------------------------------ Gather cases --------------------------------------------- */ /* ----------------------------------------------------------------------------------------------------- */ -#define CASE_GATHER_FP32_1 { 2, 3, 1, 4 }, { 4, 1, 1, 1 }, { 4, 3, 1, 4 }, format::bfyx, cldnn::gather::gather_axis::along_b, data_types::f32, format::bfyx, data_types::f32, format::bfyx -#define CASE_GATHER_FP32_2 { 3, 2, 1, 2 }, { 2, 3, 1, 1 }, { 2, 3, 2, 2 }, format::bfyx, cldnn::gather::gather_axis::along_b, data_types::f32, format::bfyx, data_types::f32, format::bfyx -#define CASE_GATHER_FP32_3 { 3, 1, 1, 2 }, { 2, 1, 1, 1 }, { 3, 2, 1, 2 }, format::bfyx, cldnn::gather::gather_axis::along_f, data_types::f32, format::bfyx, data_types::f32, format::bfyx -#define CASE_GATHER_FP32_4 { 5, 3, 2, 2 }, { 3, 1, 1, 1 }, { 5, 2, 2, 3 }, format::bfyx, cldnn::gather::gather_axis::along_y, data_types::f32, format::bfyx, data_types::f32, format::bfyx -#define CASE_GATHER_FP32_5 { 2, 3, 1, 2 }, { 1, 3, 1, 1 }, { 2, 3, 3, 1 }, format::bfyx, cldnn::gather::gather_axis::along_y, data_types::f32, format::bfyx, data_types::f32, format::bfyx +#define CASE_GATHER_FP32_1 { 2, 3, 1, 4 }, { 4, 1, 1, 1 }, { 4, 3, 4, 1 }, 0, data_types::f32, format::bfyx, data_types::f32, format::bfyx +#define CASE_GATHER_FP32_2 { 3, 2, 1, 2 }, { 2, 3, 1, 1 }, { 2, 3, 2, 2 }, 0, data_types::f32, format::bfyx, data_types::f32, format::bfyx +#define CASE_GATHER_FP32_3 { 3, 1, 1, 2 }, { 2, 1, 1, 1 }, { 3, 2, 2, 1 }, 1, data_types::f32, format::bfyx, data_types::f32, format::bfyx +#define CASE_GATHER_FP32_4 { 5, 3, 2, 2 }, { 3, 1, 1, 1 }, { 5, 2, 3, 2 }, 2, data_types::f32, format::bfyx, data_types::f32, format::bfyx +#define CASE_GATHER_FP32_5 { 2, 3, 1, 2 }, { 1, 3, 1, 1 }, { 2, 3, 1, 3 }, 2, data_types::f32, format::bfyx, data_types::f32, format::bfyx -#define CASE_GATHER_FP16_1 { 2, 3, 1, 4 }, { 4, 1, 1, 1 }, { 4, 3, 1, 4 }, format::bfyx, cldnn::gather::gather_axis::along_b, data_types::f16, format::bfyx, data_types::f16, format::bfyx -#define CASE_GATHER_FP16_2 { 3, 2, 1, 2 }, { 2, 3, 1, 1 }, { 2, 3, 2, 2 }, format::bfyx, cldnn::gather::gather_axis::along_b, data_types::f16, format::bfyx, data_types::f16, format::bfyx -#define CASE_GATHER_FP16_3 { 3, 1, 1, 2 }, { 2, 1, 1, 1 }, { 3, 2, 1, 2 }, format::bfyx, cldnn::gather::gather_axis::along_f, data_types::f16, format::bfyx, data_types::f16, format::bfyx -#define CASE_GATHER_FP16_4 { 5, 3, 2, 2 }, { 3, 1, 1, 1 }, { 5, 2, 2, 3 }, format::bfyx, cldnn::gather::gather_axis::along_y, data_types::f16, format::bfyx, data_types::f16, format::bfyx -#define CASE_GATHER_FP16_5 { 2, 3, 1, 2 }, { 1, 3, 1, 1 }, { 2, 3, 3, 1 }, format::bfyx, cldnn::gather::gather_axis::along_y, data_types::f16, format::bfyx, data_types::f16, format::bfyx +#define CASE_GATHER_FP16_1 { 2, 3, 1, 4 }, { 4, 1, 1, 1 }, { 4, 3, 4, 1 }, 0, data_types::f16, format::bfyx, data_types::f16, format::bfyx +#define CASE_GATHER_FP16_2 { 3, 2, 1, 2 }, { 2, 3, 1, 1 }, { 2, 3, 2, 2 }, 0, data_types::f16, format::bfyx, data_types::f16, format::bfyx +#define CASE_GATHER_FP16_3 { 3, 1, 1, 2 }, { 2, 1, 1, 1 }, { 3, 2, 2, 1 }, 1, data_types::f16, format::bfyx, data_types::f16, format::bfyx +#define CASE_GATHER_FP16_4 { 5, 3, 2, 2 }, { 3, 1, 1, 1 }, { 5, 2, 3, 2 }, 2, data_types::f16, format::bfyx, data_types::f16, format::bfyx +#define CASE_GATHER_FP16_5 { 2, 3, 1, 2 }, { 1, 3, 1, 1 }, { 2, 3, 1, 3 }, 2, data_types::f16, format::bfyx, data_types::f16, format::bfyx -#define CASE_GATHER_5D_FP32_1 { 2, 3, 1, 4, 1 }, { 4, 1, 1, 1 }, { 4, 3, 1, 4, 1 }, format::bfzyx, cldnn::gather::gather_axis::along_b, data_types::f32, format::bfzyx, data_types::f32, format::bfzyx -#define CASE_GATHER_5D_FP32_2 { 2, 3, 2, 2, 2 }, { 2, 1, 1, 1 }, { 2, 2, 2, 2, 2 }, format::bfzyx, cldnn::gather::gather_axis::along_f, data_types::f32, format::bfzyx, data_types::f32, format::bfzyx -#define CASE_GATHER_5D_FP32_3 { 5, 3, 2, 2, 2 }, { 3, 1, 1, 1 }, { 5, 3, 2, 3, 2 }, format::bfzyx, cldnn::gather::gather_axis::along_y, data_types::f32, format::bfzyx, data_types::f32, format::bfzyx -#define CASE_GATHER_5D_FP32_4 { 2, 3, 1, 4, 4 }, { 2, 1, 1, 1 }, { 2, 3, 1, 4, 2 }, format::bfzyx, cldnn::gather::gather_axis::along_z, data_types::f32, format::bfzyx, data_types::f32, format::bfzyx -#define CASE_GATHER_5D_FP32_5 { 3, 1, 5, 2, 1 }, { 2, 1, 1, 1 }, { 3, 1, 2, 2, 1 }, format::bfzyx, cldnn::gather::gather_axis::along_x, data_types::f32, format::bfzyx, data_types::f32, format::bfzyx +#define CASE_GATHER_5D_FP32_1 { 2, 3, 1, 4, 1 }, { 4, 1, 1, 1 }, { 4, 3, 1, 4, 1 }, 0, data_types::f32, format::bfzyx, data_types::f32, format::bfzyx +#define CASE_GATHER_5D_FP32_2 { 2, 3, 2, 2, 2 }, { 2, 1, 1, 1 }, { 2, 2, 2, 2, 2 }, 1, data_types::f32, format::bfzyx, data_types::f32, format::bfzyx +#define CASE_GATHER_5D_FP32_3 { 5, 3, 2, 2, 2 }, { 3, 1, 1, 1 }, { 5, 3, 2, 3, 2 }, 3, data_types::f32, format::bfzyx, data_types::f32, format::bfzyx +#define CASE_GATHER_5D_FP32_4 { 2, 3, 1, 4, 4 }, { 2, 1, 1, 1 }, { 2, 3, 2, 4, 1 }, 2, data_types::f32, format::bfzyx, data_types::f32, format::bfzyx +#define CASE_GATHER_5D_FP32_5 { 3, 1, 5, 2, 1 }, { 2, 1, 1, 1 }, { 3, 1, 1, 2, 2 }, 4, data_types::f32, format::bfzyx, data_types::f32, format::bfzyx -#define CASE_GATHER_5D_FP16_1 { 3, 2, 1, 2, 1 }, { 2, 1, 1, 1 }, { 2, 2, 2, 2, 1 }, format::bfzyx, cldnn::gather::gather_axis::along_b, data_types::f16, format::bfzyx, data_types::f16, format::bfzyx -#define CASE_GATHER_5D_FP16_2 { 1, 3, 1, 2, 1 }, { 2, 1, 1, 1 }, { 1, 2, 1, 2, 1 }, format::bfzyx, cldnn::gather::gather_axis::along_f, data_types::f16, format::bfzyx, data_types::f16, format::bfzyx -#define CASE_GATHER_5D_FP16_3 { 2, 3, 1, 3, 3 }, { 1, 2, 1, 1 }, { 2, 3, 1, 2, 3 }, format::bfzyx, cldnn::gather::gather_axis::along_y, data_types::f16, format::bfzyx, data_types::f16, format::bfzyx -#define CASE_GATHER_5D_FP16_4 { 3, 2, 2, 2, 2 }, { 2, 1, 1, 1 }, { 3, 2, 2, 2, 2 }, format::bfzyx, cldnn::gather::gather_axis::along_z, data_types::f16, format::bfzyx, data_types::f16, format::bfzyx -#define CASE_GATHER_5D_FP16_5 { 1, 1, 2, 1, 1 }, { 3, 1, 1, 1 }, { 1, 1, 3, 1, 1 }, format::bfzyx, cldnn::gather::gather_axis::along_x, data_types::f16, format::bfzyx, data_types::f16, format::bfzyx +#define CASE_GATHER_5D_FP16_1 { 3, 2, 1, 2, 1 }, { 2, 1, 1, 1 }, { 2, 2, 1, 2, 2 }, 0, data_types::f16, format::bfzyx, data_types::f16, format::bfzyx +#define CASE_GATHER_5D_FP16_2 { 1, 3, 1, 2, 1 }, { 2, 1, 1, 1 }, { 1, 2, 1, 2, 1 }, 1, data_types::f16, format::bfzyx, data_types::f16, format::bfzyx +#define CASE_GATHER_5D_FP16_3 { 2, 3, 1, 3, 3 }, { 1, 2, 1, 1 }, { 2, 3, 3, 2, 1 }, 3, data_types::f16, format::bfzyx, data_types::f16, format::bfzyx +#define CASE_GATHER_5D_FP16_4 { 3, 2, 2, 2, 2 }, { 2, 1, 1, 1 }, { 3, 2, 2, 2, 2 }, 2, data_types::f16, format::bfzyx, data_types::f16, format::bfzyx +#define CASE_GATHER_5D_FP16_5 { 1, 1, 2, 1, 1 }, { 3, 1, 1, 1 }, { 1, 1, 1, 1, 3 }, 4, data_types::f16, format::bfzyx, data_types::f16, format::bfzyx class gather_quantize : public GatherPrimitiveFusingTest {}; TEST_P(gather_quantize, basic) { @@ -114,7 +99,7 @@ TEST_P(gather_quantize, basic) { data("in_hi", get_mem(get_per_channel_layout(p), 1, max_random)), data("out_lo", get_mem(get_single_element_layout(p), -127)), data("out_hi", get_mem(get_single_element_layout(p), 127)), - gather("gather_prim", "input", "gather_indices", p.axis, p.out_format, p.out_shape), + gather("gather_prim", "input", "gather_indices", p.axis, p.out_shape), quantize("quantize", "gather_prim", "in_lo", "in_hi", "out_lo", "out_hi", 255, data_types::i8), reorder("reorder_bfyx", "quantize", p.default_format, data_types::f32) ); @@ -156,7 +141,7 @@ TEST_P(gather_scale_activation, basic) { input_layout("input", get_input_layout(p)), data("gather_indices", get_mem(get_indices_layout(p), 0, static_cast(get_axis_dim(p) - 1))), data("scale_data", get_mem(get_per_channel_layout(p), -10, 10)), - gather("gather_prim", "input", "gather_indices", p.axis, p.out_format, p.out_shape), + gather("gather_prim", "input", "gather_indices", p.axis, p.out_shape), activation("activation", "gather_prim", activation_func::abs), scale("scale", "activation", "scale_data"), reorder("reorder_bfyx", "scale", p.default_format, data_types::f32) diff --git a/src/plugins/intel_gpu/tests/test_cases/gather_gpu_test.cpp b/src/plugins/intel_gpu/tests/test_cases/gather_gpu_test.cpp index fd5538e1f0a..8550ae3657e 100644 --- a/src/plugins/intel_gpu/tests/test_cases/gather_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/test_cases/gather_gpu_test.cpp @@ -46,9 +46,9 @@ TEST(gather8_gpu_fp16, d323_axisY_bdim_m1) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f16, format::bfzyx, { 3, 2, 2, 4, 3} }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, { 3, 2, 1, 3 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_y; + auto input1 = engine.allocate_memory({ data_types::f16, format::bfzyx, tensor{ 3, 2, 2, 4, 3} }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 3, 2, 1, 3 } }); // Indexes + int64_t axis = 3; int64_t batch_dim = -1; bool negative_indexes = true; @@ -95,7 +95,7 @@ TEST(gather8_gpu_fp16, d323_axisY_bdim_m1) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfzyx, tensor(3, 2, 2, 3, 3), batch_dim, negative_indexes) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{3, 2, 3, 3, 2}, batch_dim, negative_indexes) ); network network(engine, topology); @@ -171,9 +171,9 @@ TEST(gather7_gpu_fp16, d222_axisX_bdim_m1) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f16, format::bfwzyx, { 2, 2, 2, 2, 2, 2} }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, { 2, 2, 1, 2 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_x; + auto input1 = engine.allocate_memory({ data_types::f16, format::bfwzyx, tensor{ 2, 2, 2, 2, 2, 2} }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 2, 2, 1, 2 } }); // Indexes + int64_t axis = 5; int64_t batch_dim = -1; set_values(input1, { @@ -202,7 +202,7 @@ TEST(gather7_gpu_fp16, d222_axisX_bdim_m1) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfwzyx, tensor(2, 2, 2, 2, 2, 2), batch_dim) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{2, 2, 2, 2, 2, 2}, batch_dim) ); network network(engine, topology); @@ -265,9 +265,9 @@ TEST(gather7_gpu_fp16, d323_axisY_bdim_m1) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f16, format::bfzyx, { 3, 2, 2, 4, 3} }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, { 3, 2, 1, 3 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_y; + auto input1 = engine.allocate_memory({ data_types::f16, format::bfzyx, tensor{ 3, 2, 2, 4, 3} }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 3, 2, 1, 3 } }); // Indexes + int64_t axis = 3; int64_t batch_dim = -1; set_values(input1, { @@ -313,7 +313,7 @@ TEST(gather7_gpu_fp16, d323_axisY_bdim_m1) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfzyx, tensor(3, 2, 2, 3, 3), batch_dim) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{3, 2, 3, 3, 2}, batch_dim) ); network network(engine, topology); @@ -383,9 +383,9 @@ TEST(gather7_gpu_fp16, d44_axisY_bdim1) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f16, format::bfyx, { 4, 3, 1, 5 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, { 4, 4, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_y; + auto input1 = engine.allocate_memory({ data_types::f16, format::bfyx, tensor{ 4, 3, 1, 5 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 4, 4, 1, 1 } }); // Indexes + int64_t axis = 2; int64_t batch_dim = 1; set_values(input1, { @@ -417,7 +417,7 @@ TEST(gather7_gpu_fp16, d44_axisY_bdim1) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(4, 3, 1, 4), batch_dim) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{4, 3, 4, 1}, batch_dim) ); network network(engine, topology); @@ -472,9 +472,9 @@ TEST(gather7_gpu_fp16, d32_axisF_bdim_m1) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f16, format::bfyx, { 3, 2, 1, 1 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, { 3, 2, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_f; + auto input1 = engine.allocate_memory({ data_types::f16, format::bfyx, tensor{ 3, 2, 1, 1 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 3, 2, 1, 1 } }); // Indexes + int64_t axis = 1; size_t batch_dim = -1; set_values(input1, { @@ -492,7 +492,7 @@ TEST(gather7_gpu_fp16, d32_axisF_bdim_m1) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(3, 2, 1, 1), batch_dim) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{3, 2, 1, 1}, batch_dim) ); network network(engine, topology); @@ -535,9 +535,9 @@ TEST(gather7_gpu_fp16, d32_axisF_bdim1) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f16, format::bfyx, { 3, 2, 1, 1 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, { 3, 2, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_f; + auto input1 = engine.allocate_memory({ data_types::f16, format::bfyx, tensor{ 3, 2, 1, 1 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 3, 2, 1, 1 } }); // Indexes + int64_t axis = 1; int64_t batch_dim = 1; set_values(input1, { @@ -555,7 +555,7 @@ TEST(gather7_gpu_fp16, d32_axisF_bdim1) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(3, 2, 1, 1), batch_dim) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{3, 2, 1, 1}, batch_dim) ); network network(engine, topology); @@ -597,9 +597,9 @@ TEST(gather7_gpu_fp16, d32_axisF_bdim0) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f16, format::bfyx, { 3, 2, 1, 1 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, { 3, 2, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_f; + auto input1 = engine.allocate_memory({ data_types::f16, format::bfyx, tensor{ 3, 2, 1, 1 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 3, 2, 1, 1 } }); // Indexes + int64_t axis = 1; size_t batch_dim = 0; set_values(input1, { @@ -617,7 +617,7 @@ TEST(gather7_gpu_fp16, d32_axisF_bdim0) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(3, 3, 1, 2), batch_dim) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{3, 3, 2, 1}, batch_dim) ); network network(engine, topology); @@ -669,7 +669,7 @@ TEST(gather_gpu_fp16, d14_axisB) { auto input1 = engine.allocate_memory({ data_types::f16, format::bfyx, { 2, 2, 1, 1 } }); // Dictionary auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, { 1, 4, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_b; + int64_t axis = 0; set_values(input1, { FLOAT16(1.0f), FLOAT16(2.0f), @@ -685,7 +685,7 @@ TEST(gather_gpu_fp16, d14_axisB) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(1, 4, 1, 2)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{1, 4, 2, 1}) ); network network(engine, topology); @@ -726,9 +726,9 @@ TEST(gather_gpu_fp16, d222_axisB) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f16, format::bfyx, { 3, 2, 1, 2 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, { 2, 2, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_b; + auto input1 = engine.allocate_memory({ data_types::f16, format::bfyx, tensor{ 3, 2, 1, 2 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 2, 2, 1, 1 } }); // Indexes + int64_t axis = 0; set_values(input1, { FLOAT16(1.f), FLOAT16(2.f), FLOAT16(3.f), @@ -747,7 +747,7 @@ TEST(gather_gpu_fp16, d222_axisB) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(2, 2, 2, 2)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{2, 2, 2, 2}) ); network network(engine, topology); @@ -788,9 +788,9 @@ TEST(gather_gpu_fp16, d22_axisY) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f16, format::bfyx, { 2, 2, 1, 3 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, { 2, 2, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_y; + auto input1 = engine.allocate_memory({ data_types::f16, format::bfyx, tensor{ 2, 2, 1, 3 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 2, 2, 1, 1 } }); // Indexes + int64_t axis = 2; set_values(input1, { FLOAT16(1.f), FLOAT16(2.f), FLOAT16(3.f), @@ -808,7 +808,7 @@ TEST(gather_gpu_fp16, d22_axisY) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(2, 2, 2, 2)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{2, 2, 2, 2}) ); network network(engine, topology); @@ -849,9 +849,9 @@ TEST(gather_gpu_fp16, d22_axisF) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f16, format::bfyx, { 2, 3, 1, 2 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, { 2, 2, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_f; + auto input1 = engine.allocate_memory({ data_types::f16, format::bfyx, tensor{ 2, 3, 1, 2 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 2, 2, 1, 1 } }); // Indexes + int64_t axis = 1; set_values(input1, { FLOAT16(1.f), FLOAT16(2.f), FLOAT16(3.f), @@ -869,7 +869,7 @@ TEST(gather_gpu_fp16, d22_axisF) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(2, 2, 2, 2)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{2, 2, 2, 2}) ); network network(engine, topology); @@ -909,9 +909,9 @@ TEST(gather_gpu_fp32, d14_axisB) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f32, format::bfyx, { 2, 2, 1, 1 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, { 1, 4, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_b; + auto input1 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 2, 2, 1, 1 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 1, 4, 1, 1 } }); // Indexes + int64_t axis = 0; set_values(input1, { 1.0f, 2.0f, @@ -927,7 +927,7 @@ TEST(gather_gpu_fp32, d14_axisB) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(1, 4, 1, 2)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{1, 4, 2, 1}) ); network network(engine, topology); @@ -968,9 +968,9 @@ TEST(gather_gpu_fp32, d222_axisB) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f32, format::bfyx, { 3, 2, 1, 2 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, { 2, 2, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_b; + auto input1 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 3, 2, 1, 2 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 2, 2, 1, 1 } }); // Indexes + int64_t axis = 0; set_values(input1, { 1.f, 2.f, 3.f, @@ -988,7 +988,7 @@ TEST(gather_gpu_fp32, d222_axisB) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(2, 2, 2, 2)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{2, 2, 2, 2}) ); network network(engine, topology); @@ -1029,9 +1029,9 @@ TEST(gather_gpu_fp32, d22_axisY) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f32, format::bfyx, { 2, 2, 1, 3 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, { 2, 2, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_y; + auto input1 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 2, 2, 1, 3 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 2, 2, 1, 1 } }); // Indexes + int64_t axis = 2; set_values(input1, { 1.f, 2.f, 3.f, @@ -1049,7 +1049,7 @@ TEST(gather_gpu_fp32, d22_axisY) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(2, 2, 2, 2)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{2, 2, 2, 2}) ); network network(engine, topology); @@ -1090,9 +1090,9 @@ TEST(gather_gpu_fp32, d22_axisF) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f32, format::bfyx, { 2, 3, 1, 2 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, { 2, 2, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_f; + auto input1 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 2, 3, 1, 2 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 2, 2, 1, 1 } }); // Indexes + int64_t axis = 1; set_values(input1, { 1.f, 2.f, 3.f, @@ -1110,7 +1110,7 @@ TEST(gather_gpu_fp32, d22_axisF) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(2, 2, 2, 2)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{2, 2, 2, 2}) ); network network(engine, topology); @@ -1151,9 +1151,9 @@ TEST(gather_gpu_int32, d22_axisF) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::i32, format::bfyx, { 2, 3, 1, 2 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, { 2, 2, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_f; + auto input1 = engine.allocate_memory({ data_types::i32, format::bfyx, tensor{ 2, 3, 1, 2 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, tensor{ 2, 2, 1, 1 } }); // Indexes + int64_t axis = 1; set_values(input1, { 1, 2, 3, @@ -1171,7 +1171,7 @@ TEST(gather_gpu_int32, d22_axisF) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(2, 2, 2, 2)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{2, 2, 2, 2}) ); network network(engine, topology); @@ -1211,9 +1211,9 @@ TEST(gather_gpu_int32, d14_axisB) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::i32, format::bfyx, { 2, 2, 1, 1 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, { 1, 4, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_b; + auto input1 = engine.allocate_memory({ data_types::i32, format::bfyx, tensor{ 2, 2, 1, 1 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, tensor{ 1, 4, 1, 1 } }); // Indexes + int64_t axis = 0; set_values(input1, { 1, 2, @@ -1229,7 +1229,7 @@ TEST(gather_gpu_int32, d14_axisB) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(1, 4, 1, 2)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{1, 4, 2, 1}) ); network network(engine, topology); @@ -1270,9 +1270,9 @@ TEST(gather_gpu_int32, d222_axisB) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::i32, format::bfyx, { 3, 2, 1, 2 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, { 2, 2, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_b; + auto input1 = engine.allocate_memory({ data_types::i32, format::bfyx, tensor{ 3, 2, 1, 2 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, tensor{ 2, 2, 1, 1 } }); // Indexes + int64_t axis = 0; set_values(input1, { 1, 2, 3, @@ -1290,7 +1290,7 @@ TEST(gather_gpu_int32, d222_axisB) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(2, 2, 2, 2)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{2, 2, 2, 2}) ); network network(engine, topology); @@ -1331,9 +1331,9 @@ TEST(gather_gpu_int32, d22_axisY) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::i32, format::bfyx, { 2, 2, 1, 3 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, { 2, 2, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_y; + auto input1 = engine.allocate_memory({ data_types::i32, format::bfyx, tensor{ 2, 2, 1, 3 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, tensor{ 2, 2, 1, 1 } }); // Indexes + int64_t axis = 2; set_values(input1, { 1, 2, 3, @@ -1351,7 +1351,7 @@ TEST(gather_gpu_int32, d22_axisY) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(2, 2, 2, 2)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{2, 2, 2, 2}) ); network network(engine, topology); @@ -1395,9 +1395,9 @@ TEST(gather_gpu_fp32, d41_axisB) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f32, format::bfyx, { 2, 2, 1, 3 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, { 4, 1, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_b; + auto input1 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 2, 2, 1, 3 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, tensor{ 4, 1, 1, 1 } }); // Indexes + int64_t axis = 0; set_values(input1, { 1.f, 2.f, 3.f, @@ -1415,7 +1415,7 @@ TEST(gather_gpu_fp32, d41_axisB) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(4, 1, 3, 2)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{4, 1, 2, 3}) ); network network(engine, topology); @@ -1461,9 +1461,9 @@ TEST(gather_gpu_fp32, d41_axisF) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f32, format::bfyx, { 2, 3, 1, 2 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, { 4, 1, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_f; + auto input1 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 2, 3, 1, 2 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, tensor{ 4, 1, 1, 1 } }); // Indexes + int64_t axis = 1; set_values(input1, { 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, @@ -1478,7 +1478,7 @@ TEST(gather_gpu_fp32, d41_axisF) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(2, 4, 2, 1)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{2, 4, 1, 2}) ); network network(engine, topology); @@ -1520,9 +1520,9 @@ TEST(gather_gpu_fp32, d2_axisX) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::f32, format::bfyx, { 2, 2, 1, 1 } }); // Dictionary - auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, { 2, 1, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_x; + auto input1 = engine.allocate_memory({ data_types::f32, format::bfyx, tensor{ 2, 2, 1, 1 } }); // Dictionary + auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, tensor{ 2, 1, 1, 1 } }); // Indexes + int64_t axis = 3; set_values(input1, { 1.f, 2.f, @@ -1537,7 +1537,7 @@ TEST(gather_gpu_fp32, d2_axisX) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(2, 2, 2, 1)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{2, 2, 1, 2}) ); network network(engine, topology); @@ -1570,9 +1570,9 @@ TEST(gather_gpu_fp32, 322_axisF) { auto& engine = get_test_engine(); - auto input1 = engine.allocate_memory({ data_types::i32, format::bfyx, { 3, 3, 1, 1 } }); // data - auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, { 2, 2, 1, 1 } }); // Indexes - auto axis = cldnn::gather::gather_axis::along_f; + auto input1 = engine.allocate_memory({ data_types::i32, format::bfyx, tensor{ 3, 3, 1, 1 } }); // data + auto input2 = engine.allocate_memory({ data_types::i32, format::bfyx, tensor{ 2, 2, 1, 1 } }); // Indexes + int64_t axis = 1; set_values(input1, { 0, 1, 2, 10, 11, 12, 20, 21, 22 @@ -1587,7 +1587,7 @@ TEST(gather_gpu_fp32, 322_axisF) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(3, 2, 1, 2)) + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{3, 2, 2, 1}) ); network network(engine, topology); @@ -1619,9 +1619,9 @@ TEST(gather_gpu_u8, 322_axisF) { auto &engine = get_test_engine(); - auto input1 = engine.allocate_memory({data_types::u8, format::bfyx, {3, 3, 1, 1}}); // data - auto input2 = engine.allocate_memory({data_types::i32, format::bfyx, {2, 2, 1, 1}}); // Indexes - auto axis = cldnn::gather::gather_axis::along_f; + auto input1 = engine.allocate_memory({data_types::u8, format::bfyx, tensor{3, 3, 1, 1}}); // data + auto input2 = engine.allocate_memory({data_types::i32, format::bfyx, tensor{2, 2, 1, 1}}); // Indexes + int64_t axis = 1; set_values(input1, {0, 1, 2, 10, 11, 12, 20, 21, 22}); @@ -1632,7 +1632,7 @@ TEST(gather_gpu_u8, 322_axisF) { topology.add(input_layout("InputDictionary", input1->get_layout())); topology.add(input_layout("InputText", input2->get_layout())); topology.add( - gather("gather", "InputDictionary", "InputText", axis, format::bfyx, tensor(3, 2, 1, 2))); + gather("gather", "InputDictionary", "InputText", axis, ov::Shape{3, 2, 2, 1})); network network(engine, topology); From d3060d4bccc5cd5955768f2f33a2c3d9eeff29e4 Mon Sep 17 00:00:00 2001 From: Elizaveta Lobanova Date: Thu, 31 Mar 2022 20:03:51 +0300 Subject: [PATCH 39/39] [GNA] Fixed handling of unaligned crop layer (#11316) --- src/plugins/intel_gna/gna_graph_compiler.cpp | 53 ++++----------- .../intel_gna/layers/gna_crop_layer.hpp | 43 ++++++++++++ .../intel_gna/layers/gna_layer_info.hpp | 7 +- .../subgraph_tests/stridedslice_concat.cpp | 38 +++++++++++ .../subgraph_tests/stridedslice_concat.hpp | 15 +++++ .../subgraph/stridedslice_concat.hpp | 43 ++++++++++++ .../src/subgraph/stridedslice_concat.cpp | 66 +++++++++++++++++++ src/tests/unit/gna/gna_get_crop_params.cpp | 60 +++++++++++++++++ 8 files changed, 284 insertions(+), 41 deletions(-) create mode 100644 src/tests/functional/plugin/gna/shared_tests_instances/subgraph_tests/stridedslice_concat.cpp create mode 100644 src/tests/functional/plugin/shared/include/subgraph_tests/stridedslice_concat.hpp create mode 100644 src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/stridedslice_concat.hpp create mode 100644 src/tests/functional/shared_test_classes/src/subgraph/stridedslice_concat.cpp create mode 100644 src/tests/unit/gna/gna_get_crop_params.cpp diff --git a/src/plugins/intel_gna/gna_graph_compiler.cpp b/src/plugins/intel_gna/gna_graph_compiler.cpp index 2d6954816bc..ccddf42ae7d 100644 --- a/src/plugins/intel_gna/gna_graph_compiler.cpp +++ b/src/plugins/intel_gna/gna_graph_compiler.cpp @@ -1053,38 +1053,11 @@ void GNAGraphCompiler::CropPrimitive(InferenceEngine::CNNLayerPtr layer) { IE_ASSERT(!layer->insData.empty()); auto inputs = layer->insData.begin()->lock(); - IE_ASSERT(!cropLayer->axis.empty()); - IE_ASSERT(cropLayer->axis.size() == cropLayer->dim.size()); - IE_ASSERT(cropLayer->axis.size() == cropLayer->offset.size()); - - std::vector axis, dim, offset; - for (int n = 0; n < cropLayer->axis.size(); n++) { - uint32_t input_dim = GetDataDimSize(inputs, inputs->getDims().size() - cropLayer->axis[n]); - // Exclude crop layer components that do nothing - if (cropLayer->offset[n] == 0 && cropLayer->dim[n] == input_dim) { - continue; - } - axis.push_back(cropLayer->axis[n]); - dim.push_back(cropLayer->dim[n]); - offset.push_back(cropLayer->offset[n]); - } - - if (axis.size() != 1) { - THROW_GNA_EXCEPTION << - "Crop layer does not support the number of (non-trivial) cropped dimensions more than 1, provided: " - << axis.size() << "."; - } - - size_t cropOffset = offset.front() * cropLayer->precision.size(); - size_t cropOutputSize = dim.front() * cropLayer->precision.size(); - const uint32_t noOfInputsDivisor = gnaFlags->input_low_precision ? - GNALimitations::noOfInputsLowPrecDivisor : GNALimitations::noOfInputsDivisor; - - // fix for crop on tensor dim > 2D - for (int n = axis[0]+1; n < cropLayer->dim.size(); n++) { - cropOffset *= cropLayer->dim[n]; - cropOutputSize *= cropLayer->dim[n]; - } + size_t cropOffset, cropOutputSize; + std::vector axis; + std::tie(cropOffset, cropOutputSize, axis) = GetCropParams(cropLayer); + size_t cropOffsetBytes = cropOffset * cropLayer->precision.size(); + size_t cropOutputSizeBytes = cropOutputSize * cropLayer->precision.size(); if (!LayerInfo(cropLayer).isCropAffined()) { // leave crop as it is @@ -1099,13 +1072,13 @@ void GNAGraphCompiler::CropPrimitive(InferenceEngine::CNNLayerPtr layer) { } // calculate index idx for connectInput last parameter - connectInput(layer, &cropLayerInfo->second.gna_ptr, cropOutputSize + cropOffset, cropOffset, 0); + connectInput(layer, &cropLayerInfo->second.gna_ptr, cropOutputSizeBytes + cropOffsetBytes, cropOffsetBytes, 0); // cases for certain output layers for (auto&& outLayer : getInputTo(layer->outData.front())) { auto& nextLayer = outLayer.second; if (LayerInfo(nextLayer).isConcat()) { - connectOutput(layer, &cropLayerInfo->second.gna_ptr, cropOutputSize); + connectOutput(layer, &cropLayerInfo->second.gna_ptr, cropOutputSizeBytes); } } } else { @@ -1119,10 +1092,12 @@ void GNAGraphCompiler::CropPrimitive(InferenceEngine::CNNLayerPtr layer) { } // TODO: add unit tests for 4d crops blobs - uint32_t num_rows_in = GetDataDimSize(inputs, inputs->getDims().size() - axis.front()); + uint32_t num_rows_in = InferenceEngine::details::product(begin(inputs->getDims()), end(inputs->getDims())); uint32_t num_columns_in = 1; - uint32_t num_rows_out = GetDataDimSize(outputs, inputs->getDims().size() - axis.front()); + uint32_t num_rows_out = InferenceEngine::details::product(begin(outputs->getDims()), end(outputs->getDims())); + const uint32_t noOfInputsDivisor = gnaFlags->input_low_precision ? + GNALimitations::noOfInputsLowPrecDivisor : GNALimitations::noOfInputsDivisor; uint32_t num_padding = ALIGN(num_rows_in, noOfInputsDivisor) - num_rows_in; void* ptr_inputs = nullptr; @@ -1158,7 +1133,7 @@ void GNAGraphCompiler::CropPrimitive(InferenceEngine::CNNLayerPtr layer) { connectInput(layer, ptr_inputs, num_data_bytes_in, 0, 0); connectOutput(layer, ptr_outputs, num_data_bytes_out); - FillWeightOfAligningFilter(layer, ptr_weights, offset.front(), (quantized == nullptr) ? false : true); + FillWeightOfAligningFilter(layer, ptr_weights, cropOffset, (quantized == nullptr) ? false : true); (quantized == nullptr) ? gnamem->readonly().push_value(layer, ptr_biases, 0.0f, num_rows_out, 64) : @@ -1589,8 +1564,8 @@ void GNAGraphCompiler::FillWeightOfAligningFilter(InferenceEngine::CNNLayerPtr l auto outputs = *layer->outData.begin(); auto inputs = layer->insData.begin()->lock(); - uint32_t num_rows_in = InferenceEngine::details::product(++begin(inputs->getDims()), end(inputs->getDims())); - uint32_t num_rows_out = InferenceEngine::details::product(++begin(outputs->getDims()), end(outputs->getDims())); + uint32_t num_rows_in = InferenceEngine::details::product(begin(inputs->getDims()), end(inputs->getDims())); + uint32_t num_rows_out = InferenceEngine::details::product(begin(outputs->getDims()), end(outputs->getDims())); if (!ptrWeights) { THROW_GNA_EXCEPTION << "Weights memory is not allocated!!!"; diff --git a/src/plugins/intel_gna/layers/gna_crop_layer.hpp b/src/plugins/intel_gna/layers/gna_crop_layer.hpp index 3d7abe23507..7c8741663e0 100644 --- a/src/plugins/intel_gna/layers/gna_crop_layer.hpp +++ b/src/plugins/intel_gna/layers/gna_crop_layer.hpp @@ -5,6 +5,7 @@ #pragma once #include +#include "gna_graph_tools.hpp" namespace GNAPluginNS { class GNACropLayer { @@ -16,9 +17,51 @@ public: {} InferenceEngine::CNNLayerPtr getCrop() { return cropLayer; } + /** * pointer to gna croped memory beginning */ void *gna_ptr = nullptr; }; + +/** + * @brief returns parameters extracted from Crop layer: elements offset, elements output size and axes + * @param cropLayer pointer to a Crop layer + */ +inline std::tuple> GetCropParams(InferenceEngine::CropLayer* cropLayer) { + IE_ASSERT(!cropLayer->axis.empty()); + IE_ASSERT(cropLayer->axis.size() == cropLayer->dim.size()); + IE_ASSERT(cropLayer->axis.size() == cropLayer->offset.size()); + + std::vector axis, dim, offset; + auto inputs = cropLayer->insData.begin()->lock(); + for (int n = 0; n < cropLayer->axis.size(); n++) { + uint32_t input_dim = GetDataDimSize(inputs, inputs->getDims().size() - cropLayer->axis[n]); + // Exclude crop layer components that do nothing + if (cropLayer->offset[n] == 0 && cropLayer->dim[n] == input_dim) { + continue; + } + axis.push_back(cropLayer->axis[n]); + dim.push_back(cropLayer->dim[n]); + offset.push_back(cropLayer->offset[n]); + } + + if (axis.size() != 1) { + THROW_GNA_EXCEPTION << + "Crop layer does not support the number of (non-trivial) cropped dimensions more than 1, provided: " + << axis.size() << "."; + } + + size_t cropOffset = offset.front(); + size_t cropOutputSize = dim.front(); + + // fix for crop on tensor dim > 2D + for (int n = axis[0]+1; n < cropLayer->dim.size(); n++) { + cropOffset *= cropLayer->dim[n]; + cropOutputSize *= cropLayer->dim[n]; + } + + return std::make_tuple(cropOffset, cropOutputSize, axis); +} + } // namespace GNAPluginNS diff --git a/src/plugins/intel_gna/layers/gna_layer_info.hpp b/src/plugins/intel_gna/layers/gna_layer_info.hpp index 4848aac8f0b..2c3830130ec 100644 --- a/src/plugins/intel_gna/layers/gna_layer_info.hpp +++ b/src/plugins/intel_gna/layers/gna_layer_info.hpp @@ -17,6 +17,7 @@ #include #include #include "ops/pwl.hpp" +#include "layers/gna_crop_layer.hpp" namespace GNAPluginNS { @@ -363,8 +364,10 @@ class LayerInfo { // currently crop layer only supports 2 bytes in int16 and int8 mode. // In fp32 mode this is not necessary but is useful for testing auto bytesPerCropElement = 2; - size_t cropOffset = cropLayer->offset.back() * bytesPerCropElement; - return (ALIGN64(cropOffset) != cropOffset); + size_t offset; + std::tie(offset, std::ignore, std::ignore) = GetCropParams(cropLayer); + size_t bytesOffset = offset * bytesPerCropElement; + return (ALIGN64(bytesOffset) != bytesOffset); } return false; } diff --git a/src/tests/functional/plugin/gna/shared_tests_instances/subgraph_tests/stridedslice_concat.cpp b/src/tests/functional/plugin/gna/shared_tests_instances/subgraph_tests/stridedslice_concat.cpp new file mode 100644 index 00000000000..59402800732 --- /dev/null +++ b/src/tests/functional/plugin/gna/shared_tests_instances/subgraph_tests/stridedslice_concat.cpp @@ -0,0 +1,38 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include "common_test_utils/test_constants.hpp" +#include "subgraph_tests/stridedslice_concat.hpp" + +using namespace SubgraphTestsDefinitions; + +namespace { +const std::vector netPrecisions = { + InferenceEngine::Precision::FP32, + InferenceEngine::Precision::FP16 +}; + +const std::vector> configs = { + { + {"GNA_DEVICE_MODE", "GNA_SW_EXACT"}, + } +}; + +const std::vector sliceParams = { + {{8, 16}, {1, 16}, {2, 16}, {1, 1}, {0, 1}, {0, 1}}, + {{1, 16}, {1, 1}, {1, 2}, {1, 1}, {1, 0}, {1, 0}}, + {{1, 8, 16}, {1, 1, 16}, {1, 2, 16}, {1, 1, 1}, {1, 0, 1}, {1, 0, 1}}, + {{8, 25}, {3, 25}, {4, 25}, {1, 1}, {0, 1}, {0, 1}} +}; + +INSTANTIATE_TEST_SUITE_P(smoke_SliceConcatTest, SliceConcatTest, + ::testing::Combine( + ::testing::ValuesIn(netPrecisions), + ::testing::Values(CommonTestUtils::DEVICE_GNA), + ::testing::ValuesIn(configs), + ::testing::ValuesIn(sliceParams)), + SliceConcatTest::getTestCaseName); +} // namespace diff --git a/src/tests/functional/plugin/shared/include/subgraph_tests/stridedslice_concat.hpp b/src/tests/functional/plugin/shared/include/subgraph_tests/stridedslice_concat.hpp new file mode 100644 index 00000000000..bfe564fa161 --- /dev/null +++ b/src/tests/functional/plugin/shared/include/subgraph_tests/stridedslice_concat.hpp @@ -0,0 +1,15 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "shared_test_classes/subgraph/stridedslice_concat.hpp" + +namespace SubgraphTestsDefinitions { + +TEST_P(SliceConcatTest, CompareWithRefImpl) { + Run(); +}; + +} // namespace SubgraphTestsDefinitions \ No newline at end of file diff --git a/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/stridedslice_concat.hpp b/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/stridedslice_concat.hpp new file mode 100644 index 00000000000..49be793b33b --- /dev/null +++ b/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/stridedslice_concat.hpp @@ -0,0 +1,43 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include + +#include "shared_test_classes/base/layer_test_utils.hpp" +#include "ngraph_functions/builders.hpp" +#include "ngraph_functions/utils/ngraph_helpers.hpp" + +namespace SubgraphTestsDefinitions { + +typedef std::tuple< + std::vector, // Input shape + std::vector, // Begin + std::vector, // End + std::vector, // Strides + std::vector, // Begin mask + std::vector // End mask +> StridedSliceParams; + +typedef std::tuple< + InferenceEngine::Precision, // Network Precision + std::string, // Target Device + std::map, // Configuration + StridedSliceParams // StridedSlice parameters +> SliceConcatParams; + +class SliceConcatTest : public testing::WithParamInterface, + virtual public LayerTestsUtils::LayerTestsCommon { +public: + static std::string getTestCaseName(const testing::TestParamInfo& obj); + +protected: + void SetUp() override; +}; + +} // namespace SubgraphTestsDefinitions diff --git a/src/tests/functional/shared_test_classes/src/subgraph/stridedslice_concat.cpp b/src/tests/functional/shared_test_classes/src/subgraph/stridedslice_concat.cpp new file mode 100644 index 00000000000..ff0d8cbd950 --- /dev/null +++ b/src/tests/functional/shared_test_classes/src/subgraph/stridedslice_concat.cpp @@ -0,0 +1,66 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "shared_test_classes/subgraph/stridedslice_concat.hpp" +#include "ngraph_functions/builders.hpp" + +namespace SubgraphTestsDefinitions { + +std::string SliceConcatTest::getTestCaseName(const testing::TestParamInfo& obj) { + InferenceEngine::Precision netPrecision; + std::string targetDevice; + std::map configuration; + StridedSliceParams sliceParams; + std::tie(netPrecision, targetDevice, configuration, sliceParams) = obj.param; + std::vector inputShape, begin, end, strides, beginMask, endMask; + std::tie(inputShape, begin, end, strides, beginMask, endMask) = sliceParams; + + std::ostringstream result; + result << "netPRC=" << netPrecision.name() << "_"; + result << "targetDevice=" << targetDevice; + for (auto const& configItem : configuration) { + result << "_configItem=" << configItem.first << "_" << configItem.second; + } + result << "IS=" << CommonTestUtils::vec2str(inputShape) << "_"; + result << "B=" << CommonTestUtils::vec2str(begin) << "_"; + result << "E=" << CommonTestUtils::vec2str(end) << "_"; + result << "S=" << CommonTestUtils::vec2str(strides) << "_"; + result << "BM=" << CommonTestUtils::vec2str(beginMask) << "_"; + result << "EM=" << CommonTestUtils::vec2str(endMask) << "_"; + return result.str(); +} + +void SliceConcatTest::SetUp() { + InferenceEngine::Precision netPrecision; + std::map tempConfig; + StridedSliceParams sliceParams; + std::tie(netPrecision, targetDevice, tempConfig, sliceParams) = this->GetParam(); + configuration.insert(tempConfig.begin(), tempConfig.end()); + std::vector inputShape, begin, end, strides, beginMask, endMask; + std::tie(inputShape, begin, end, strides, beginMask, endMask) = sliceParams; + + auto ngPrc = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(netPrecision); + size_t input_size = std::accumulate(std::begin(inputShape), std::end(inputShape), 1, std::multiplies()); + auto params = ngraph::builder::makeParams(ngPrc, {{1, input_size}}); + + ngraph::Output input = params[0]; + if (inputShape[0] != 1 || inputShape.size() != 2) { + input = std::make_shared(params[0], + ngraph::builder::makeConstant(ngraph::element::i64, ngraph::Shape{inputShape.size()}, inputShape), false); + } + + auto ss = ngraph::builder::makeStridedSlice(input, begin, end, strides, ngPrc, beginMask, endMask, + std::vector(inputShape.size(), 0), + std::vector(inputShape.size(), 0), + std::vector(inputShape.size(), 0)); + + ngraph::Shape const_shape(inputShape.size(), 1); + const_shape.back() = 32; + auto const_input = ngraph::builder::makeConstant(ngPrc, const_shape, std::vector{}, true); + auto concat = ngraph::builder::makeConcat({const_input, ss}, inputShape.size() - 1); + + function = std::make_shared(concat, params, "StridedSliceConcatTest"); +} + +} // namespace SubgraphTestsDefinitions diff --git a/src/tests/unit/gna/gna_get_crop_params.cpp b/src/tests/unit/gna/gna_get_crop_params.cpp new file mode 100644 index 00000000000..e2f714f219a --- /dev/null +++ b/src/tests/unit/gna/gna_get_crop_params.cpp @@ -0,0 +1,60 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include +// to suppress deprecated definition errors +#define IMPLEMENT_INFERENCE_ENGINE_PLUGIN +#include +#include + +namespace { + +typedef std::tuple< + std::vector, // Input shape + std::vector, // Output shape + std::vector, // Axes + std::vector, // Offset + size_t, // Offset in flatten data + size_t, // Output size in flatten data + std::vector // Axes which arre not skipped +> CropParams; + +const std::vector crop_params_vector = { + {{8, 16}, {1, 16}, {0, 1}, {1, 0}, 16, 16, {0}}, + {{5, 24}, {1, 24}, {0, 1}, {2, 0}, 48, 24, {0}}, + {{8, 16}, {2, 16}, {0, 1}, {1, 0}, 16, 32, {0}}, + {{1, 16}, {1, 8}, {0, 1}, {0, 5}, 5, 8, {1}}, + {{1, 8, 16}, {1, 1, 16}, {0, 1, 2}, {0, 1, 0}, 16, 16, {1}}, + {{1, 1, 8, 16}, {1, 1, 1, 16}, {0, 1, 2, 3}, {0, 0, 1, 0}, 16, 16, {2}} +}; + +TEST(GetCropParamsTest, testGetCropParams) { + InferenceEngine::LayerParams attrs = {"Crop", "Crop", InferenceEngine::Precision::FP32}; + for (const auto& crop_params : crop_params_vector) { + std::vector in_shape; + std::vector orig_out_shape, orig_axes, orig_offset; + size_t result_offset, result_out_size; + std::vector result_axes; + std::tie(in_shape, orig_out_shape, orig_axes, orig_offset, result_offset, result_out_size, result_axes) = crop_params; + + auto crop_layer = std::make_shared(attrs); + auto layout = in_shape.size() == 2 ? InferenceEngine::NC : (in_shape.size() == 3 ? InferenceEngine::CHW : InferenceEngine::NCHW); + auto data = std::make_shared("Crop_input", + InferenceEngine::TensorDesc(InferenceEngine::Precision::FP32, in_shape, layout)); + crop_layer->insData.push_back(data); + crop_layer->dim = orig_out_shape; + crop_layer->axis = orig_axes; + crop_layer->offset = orig_offset; + size_t offset, out_size; + std::vector axis; + std::tie(offset, out_size, axis) = GNAPluginNS::GetCropParams(crop_layer.get()); + ASSERT_EQ(offset, result_offset); + ASSERT_EQ(out_size, result_out_size); + ASSERT_EQ(axis, result_axes); + } +} + +} // namespace \ No newline at end of file