diff --git a/docs/IE_PLUGIN_DG/Building.md b/docs/IE_PLUGIN_DG/Building.md index f30d67fec22..93d6c934a03 100644 --- a/docs/IE_PLUGIN_DG/Building.md +++ b/docs/IE_PLUGIN_DG/Building.md @@ -16,22 +16,23 @@ $ cmake -DCMAKE_BUILD_TYPE=Release ../dldt Once the commands above are executed, the Inference Engine Developer Package is generated in the `dldt-release-build` folder. It consists of several files: - `InferenceEngineDeveloperPackageConfig.cmake` - the main CMake script which imports targets and provides compilation flags and CMake options. - `InferenceEngineDeveloperPackageConfig-version.cmake` - a file with a package version. - - `targets_developer.cmake` - an automatically generated file which contains all targets exported from the Deep Learning Deployment Toolkit (DLDT) build tree. This file is included by `InferenceEngineDeveloperPackageConfig.cmake` to import the following targets: + - `targets_developer.cmake` - an automatically generated file which contains all targets exported from the OpenVINO build tree. This file is included by `InferenceEngineDeveloperPackageConfig.cmake` to import the following targets: - Libraries for plugin development: - * `IE::ngraph` - shared nGraph library - * `IE::inference_engine` - shared Inference Engine library - * `IE::inference_engine_preproc` - shared library with Inference Engine preprocessing plugin - * `IE::inference_engine_plugin_api` - interface library with Inference Engine Plugin API headers - * `IE::inference_engine_lp_transformations` - shared library with low-precision transformations - * `IE::pugixml` - static Pugixml library - * `IE::xbyak` - interface library with Xbyak headers + * `IE::ngraph` - shared nGraph library + * `IE::inference_engine` - shared Inference Engine library + * `IE::inference_engine_transformations` - shared library with Inference Engine ngraph-based Transformations + * `IE::inference_engine_preproc` - shared library with Inference Engine preprocessing plugin + * `IE::inference_engine_plugin_api` - interface library with Inference Engine Plugin API headers + * `IE::inference_engine_lp_transformations` - shared library with low-precision transformations + * `IE::pugixml` - static Pugixml library + * `IE::xbyak` - interface library with Xbyak headers - Libraries for tests development: - * `IE::gtest`, `IE::gtest_main`, `IE::gmock` - Google Tests framework libraries - * `IE::commonTestUtils` - static library with common tests utilities - * `IE::funcTestUtils` - static library with functional tests utilities - * `IE::unitTestUtils` - static library with unit tests utilities - * `IE::ngraphFunctions` - static library with the set of Ngraph Functions builders - * `IE::funcSharedTests` - static library with common functional tests + * `IE::gtest`, `IE::gtest_main`, `IE::gmock` - Google Tests framework libraries + * `IE::commonTestUtils` - static library with common tests utilities + * `IE::funcTestUtils` - static library with functional tests utilities + * `IE::unitTestUtils` - static library with unit tests utilities + * `IE::ngraphFunctions` - static library with the set of `ngraph::Function` builders + * `IE::funcSharedTests` - static library with common functional tests > **Note:** it's enough just to run `cmake --build . --target ie_dev_targets` command to build only targets from the > Inference Engine Developer package. @@ -68,24 +69,20 @@ find_package(InferenceEngineDeveloperPackage REQUIRED) add_subdirectory(src) if(ENABLE_TESTS) - include(CTest) - enable_testing() + include(CTest) + enable_testing() - if(ENABLE_FUNCTIONAL_TESTS) - add_subdirectory(tests/functional) - endif() - - if(ENABLE_BEH_TESTS) - add_subdirectory(tests/behavior) - endif() + if(ENABLE_FUNCTIONAL_TESTS) + add_subdirectory(tests/functional) + endif() endif() ``` -> **NOTE**: The default values of the `ENABLE_TESTS`, `ENABLE_FUNCTIONAL_TESTS`, `ENABLE_BEH_TESTS` options are shared via the Inference Engine Developer Package and they are the same as for the main DLDT build tree. You can override them during plugin build using the command below: +> **NOTE**: The default values of the `ENABLE_TESTS`, `ENABLE_FUNCTIONAL_TESTS` options are shared via the Inference Engine Developer Package and they are the same as for the main DLDT build tree. You can override them during plugin build using the command below: - ```bash - $ cmake -DENABLE_FUNCTIONAL_TESTS=OFF -DInferenceEngineDeveloperPackage_DIR=../dldt-release-build ../template-plugin - ``` + ```bash + $ cmake -DENABLE_FUNCTIONAL_TESTS=OFF -DInferenceEngineDeveloperPackage_DIR=../dldt-release-build ../template-plugin + ``` - `src/CMakeLists.txt` to build a plugin shared library from sources: diff --git a/docs/IE_PLUGIN_DG/Doxyfile b/docs/IE_PLUGIN_DG/Doxyfile index 1f1407ea165..96ef887e7cb 100644 --- a/docs/IE_PLUGIN_DG/Doxyfile +++ b/docs/IE_PLUGIN_DG/Doxyfile @@ -869,6 +869,8 @@ EXAMPLE_PATH = ../template_plugin/src \ ../template_plugin/src/CMakeLists.txt \ ../template_plugin/tests/functional/CMakeLists.txt \ ../template_plugin/tests/functional/transformations \ + ../template_plugin/tests/functional/shared_tests_instances/ \ + ../../inference-engine/tests/functional/plugin/shared/include \ ../examples # If the value of the EXAMPLE_PATH tag contains directories, you can use the @@ -877,7 +879,8 @@ EXAMPLE_PATH = ../template_plugin/src \ # files are included. EXAMPLE_PATTERNS = *.cpp \ - *.hpp + *.hpp \ + *.txt # If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be # searched for input files to be used with the \include or \dontinclude commands @@ -2044,7 +2047,8 @@ INCLUDE_FILE_PATTERNS = # recursively expanded use the := operator instead of the = operator. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -PREDEFINED = INFERENCE_ENGINE_API \ +PREDEFINED = INFERENCE_PLUGIN_API \ + INFERENCE_ENGINE_API \ INFERENCE_ENGINE_API_CPP \ INFERENCE_ENGINE_API_CLASS \ INFERENCE_ENGINE_DEPRECATED \ diff --git a/docs/IE_PLUGIN_DG/ExecutableNetwork.md b/docs/IE_PLUGIN_DG/ExecutableNetwork.md index eecdcb6d044..a52872946c2 100644 --- a/docs/IE_PLUGIN_DG/ExecutableNetwork.md +++ b/docs/IE_PLUGIN_DG/ExecutableNetwork.md @@ -1,7 +1,7 @@ # Executable Network {#executable_network} `ExecutableNetwork` class functionality: -- Compile an InferenceEngine::ICNNNetwork instance to a hardware-specific graph representation +- Compile an InferenceEngine::ICNNNetwork instance to a backend specific graph representation - Create an arbitrary number of `InferRequest` objects - Hold some common resources shared between different instances of `InferRequest`. For example: - InferenceEngine::ExecutableNetworkInternal::_taskExecutor task executor to implement asynchronous execution @@ -19,38 +19,37 @@ Inference Engine Plugin API provides the helper InferenceEngine::ExecutableNetwo The example class has several fields: - `_requestId` - Tracks a number of created inference requests, which is used to distinguish different inference requests during profiling via the Intel® Instrumentation and Tracing Technology (ITT) library. -- `_name` - Provides a network name. - `_cfg` - Defines a configuration an executable network was compiled with. - `_plugin` - Refers to a plugin instance. +- `_function` - Keeps a reference to transformed `ngraph::Function` which is used in ngraph reference backend computations. Note, in case of other backends with backend specific graph representation `_function` has different type and represents backend specific graph or just a set of computational kernels to perform an inference. +- `_inputIndex` - maps a name of input with its index among all network inputs. +- `_outputIndex` - maps a name of output with its index among all network outputs. ### `ExecutableNetwork` Constructor with `ICNNNetwork` -This constructor accepts a generic representation of a neural network as an InferenceEngine::ICNNNetwork reference and is compiled into a hardware-specific device graph: +This constructor accepts a generic representation of a neural network as an InferenceEngine::ICNNNetwork reference and is compiled into a backend specific device graph: @snippet src/template_executable_network.cpp executable_network:ctor_cnnnetwork -The implementation `CompileGraph` is fully device-specific. +The implementation `CompileNetwork` is fully device-specific. -### `CompileGraph()` +### `CompileNetwork()` -The function accepts a const shared pointer to `const ngraph::Function` object and performs the following steps: +The function accepts a const shared pointer to `ngraph::Function` object and performs the following steps: -1. Deep copies a const object to a local object, which can later be modified. -2. Applies common and plugin-specific transformations on a copied graph to make the graph more friendly to hardware operations. For details how to write custom plugin-specific transformation, please, refer to [Writing ngraph transformations](@ref new_ngraph_transformation) guide. -3. Maps the transformed graph to a plugin-specific graph representation (for example, to MKLDNN graph for CPU). See details topics about network representation: - * [Intermediate Representation and Operation Sets](../_docs_MO_DG_IR_and_opsets.html) - * [Quantized networks](@ref quantized_networks). -4. Allocates and fills memory for graph weights. +1. Applies ngraph passes using `TransformNetwork` function, which defines plugin-specific conversion pipeline. +2. Maps the transformed graph to a backend specific graph representation (for example, to MKLDNN graph for Intel CPU). +3. Allocates and fills memory for graph weights, backend specific memory handles and so on. -@snippet src/template_executable_network.cpp executable_network:compile_graph +@snippet src/template_executable_network.cpp executable_network:map_graph -> **NOTE**: After all these steps, the hardware-specific graph is ready to create inference requests and perform inference. +> **NOTE**: After all these steps, the backend specific graph is ready to create inference requests and perform inference. ### `ExecutableNetwork` Constructor Importing from Stream -This constructor creates a hardware-specific graph by importing from a stream object: +This constructor creates a backend specific graph by importing from a stream object: -> **NOTE**: The export of hardware-specific graph is done in the `ExportImpl` method, and data formats must be the same for both import and export. +> **NOTE**: The export of backend specific graph is done in the `ExportImpl` method, and data formats must be the same for both import and export. @snippet src/template_executable_network.cpp executable_network:ctor_import_stream @@ -59,9 +58,9 @@ This constructor creates a hardware-specific graph by importing from a stream ob **Implementation details:** Base InferenceEngine::ExecutableNetworkThreadSafeDefault class implements the public InferenceEngine::ExecutableNetworkThreadSafeDefault::Export method as following: - Writes `_plugin->GetName()` to the `model` stream. -- Calls the `ExportImpl` method defined in a derived class to dump a hardware-specific graph. +- Calls the `ExportImpl` method defined in a derived class to dump a backend specific graph. -The implementation of the method should write all data to the `model` stream, which is required to import a hardware-specific graph later in the `Plugin::Import` method: +The implementation of the method should write all data to the `model` stream, which is required to import a backend specific graph later in the `Plugin::Import` method: @snippet src/template_executable_network.cpp executable_network:export_impl @@ -73,7 +72,6 @@ The method creates an asynchronous inference request and returns it. While the p - [Asynchronous inference request](@ref async_infer_request), which is a wrapper for a synchronous inference request and can run a pipeline asynchronously. Depending on a device pipeline structure, it can has one or several stages: - For single-stage pipelines, there is no need to define this method and create a class derived from InferenceEngine::AsyncInferRequestThreadSafeDefault. For single stage pipelines, a default implementation of this method creates InferenceEngine::AsyncInferRequestThreadSafeDefault wrapping a synchronous inference request and runs it asynchronously in the `_taskExecutor` executor. - For pipelines with multiple stages, such as performing some preprocessing on host, uploading input data to a device, running inference on a device, or downloading and postprocessing output data, schedule stages on several task executors to achieve better device use and performance. You can do it by creating a sufficient number of inference requests running in parallel. In this case, device stages of different inference requests are overlapped with preprocessing and postprocessing stage giving better performance. - > **IMPORTANT**: It is up to you to decide how many task executors you need to optimally execute a device pipeline. @snippet src/template_executable_network.cpp executable_network:create_infer_request diff --git a/docs/IE_PLUGIN_DG/InferRequest.md b/docs/IE_PLUGIN_DG/InferRequest.md index c7ea3794744..d5798988980 100644 --- a/docs/IE_PLUGIN_DG/InferRequest.md +++ b/docs/IE_PLUGIN_DG/InferRequest.md @@ -1,7 +1,7 @@ # Synchronous Inference Request {#infer_request} `InferRequest` class functionality: -- Allocate input and output blobs needed for a hardware-dependent network inference. +- Allocate input and output blobs needed for a backend-dependent network inference. - Define functions for inference process stages (for example, `preprocess`, `upload`, `infer`, `download`, `postprocess`). These functions can later be used to define an execution pipeline during [Asynchronous Inference Request](@ref async_infer_request) implementation. - Call inference stages one by one synchronously. @@ -20,9 +20,15 @@ The example class has several fields: - `_executableNetwork` - reference to an executable network instance. From this reference, an inference request instance can take a task executor, use counter for a number of created inference requests, and so on. - `_profilingTask` - array of the `std::array` type. Defines names for pipeline stages. Used to profile an inference pipeline execution with the Intel® instrumentation and tracing technology (ITT). -- `_inputsNCHW` - input blob map -- `_outputsNCHW` - output blob map -- Several double values to hold an execution time for pipeline stages. +- `_durations` - array of durations of each pipeline stage. +- `_networkInputBlobs` - input blob map. +- `_networkOutputBlobs` - output blob map. +- `_parameters` - `ngraph::Function` parameter operations. +- `_results` - `ngraph::Function` result operations. +- backend specific fields: + - `_inputTensors` - inputs tensors which wrap `_networkInputBlobs` blobs. They are used as inputs to backend `_executable` computational graph. + - `_outputTensors` - output tensors which wrap `_networkOutputBlobs` blobs. They are used as outputs from backend `_executable` computational graph. + - `_executable` - an executable object / backend computational graph. ### `InferRequest` Constructor @@ -30,11 +36,6 @@ The constructor initializes helper fields and calls methods which allocate blobs @snippet src/template_infer_request.cpp infer_request:ctor -The implementation of function allocating device buffers is fully device-specific and not provided in the guide. -The implementation of function allocating host buffers assumes that the `Template` device works -natively only with the InferenceEngine::NCHW input and output layout, while the user can specify the InferenceEngine::NHWC as a layout -of InferenceEngine::CNNNetwork inputs and outputs and set InferenceEngine::NHWC blobs via the InferenceEngine::InferRequest::SetBlob method. - > **NOTE**: Call InferenceEngine::CNNNetwork::getInputsInfo and InferenceEngine::CNNNetwork::getOutputsInfo to specify both layout and precision of blobs, which you can set with InferenceEngine::InferRequest::SetBlob and get with InferenceEngine::InferRequest::GetBlob. A plugin uses these hints to determine its internal layouts and precisions for input and output blobs if needed. ### `~InferRequest` Destructor @@ -51,14 +52,27 @@ Decrements a number of created inference requests: @snippet src/template_infer_request.cpp infer_request:infer_impl +#### 1. `inferPreprocess` + Below is the code of the the `inferPreprocess` method to demonstrate Inference Engine common preprocessing step handling: @snippet src/template_infer_request.cpp infer_request:infer_preprocess **Details:** * `InferImpl` must call the InferenceEngine::InferRequestInternal::execDataPreprocessing function, which executes common Inference Engine preprocessing step (for example, applies resize or color conversion operations) if it is set by the user. The output dimensions, layout and precision matches the input information set via InferenceEngine::CNNNetwork::getInputsInfo. -* To handle both InferenceEngine::NCHW and InferenceEngine::NHWC input layouts, the `TemplateInferRequest` class has the `_inputsNCHW` field, which holds blobs in the InferenceEngine::NCHW layout. During Inference Request execution, `InferImpl` copies from the input InferenceEngine::NHWC layout to `_inputsNCHW` if needed. -* The next logic of `InferImpl` works with `_inputsNCHW`. +* If `inputBlob` passed by user differs in terms of precisions from precision expected by plugin, `blobCopy` is performed which does actual precision conversion. + +#### 2. `startPipeline` + +Executes a pipeline synchronously using `_executable` object: + +@snippet src/template_infer_request.cpp infer_request:start_pipeline + +#### 3. `inferPostprocess` + +Converts output blobs if precisions of backend output blobs and blobs passed by user are different: + +@snippet src/template_infer_request.cpp infer_request:infer_postprocess ### `GetPerformanceCounts()` diff --git a/docs/IE_PLUGIN_DG/Intro.md b/docs/IE_PLUGIN_DG/Intro.md index 038f9e167b4..f8638c768b8 100644 --- a/docs/IE_PLUGIN_DG/Intro.md +++ b/docs/IE_PLUGIN_DG/Intro.md @@ -12,15 +12,15 @@ Inference Engine plugin dynamic library consists of several main components: 1. [Plugin class](@ref plugin): - Provides information about devices of a specific type. - Can create an [executable network](@ref executable_network) instance which represents a Neural - Network hardware-specific graph structure for a particular device in opposite to the InferenceEngine::ICNNNetwork - interface which is hardware-independent. + Network backend specific graph structure for a particular device in opposite to the InferenceEngine::ICNNNetwork + interface which is backend-independent. - Can import an already compiled graph structure from an input stream to an [executable network](@ref executable_network) object. 2. [Executable Network class](@ref executable_network): - Is an execution configuration compiled for a particular device and takes into account its capabilities. - Holds a reference to a particular device and a task executor for this device. - Can create several instances of [Inference Request](@ref infer_request). - - Can export an internal hardware-specific graph structure to an output stream. + - Can export an internal backend specific graph structure to an output stream. 3. [Inference Request class](@ref infer_request): - Runs an inference pipeline serially. - Can extract performance counters for an inference pipeline execution profiling. @@ -30,7 +30,7 @@ Inference Engine plugin dynamic library consists of several main components: > **NOTE**: This documentation is written based on the `Template` plugin, which demonstrates plugin development details. Find the complete code of the `Template`, which is fully compilable and up-to-date, -at `/docs_developer/template_plugin`. +at `/docs/template_plugin`. Detailed guides ----------------------- diff --git a/docs/IE_PLUGIN_DG/Plugin.md b/docs/IE_PLUGIN_DG/Plugin.md index 93f80f189d0..3c51085916a 100644 --- a/docs/IE_PLUGIN_DG/Plugin.md +++ b/docs/IE_PLUGIN_DG/Plugin.md @@ -1,5 +1,15 @@ # Plugin {#plugin} +Inference Engine Plugin usually represents a wrapper around a backend. Backends can be: +- OpenCL-like backend (e.g. clDNN library) for GPU devices. +- MKLDNN backend for Intel CPU devices. +- NVIDIA cuDNN for NVIDIA GPUs. + +The responsibility of Inference Engine Plugin: +- Initializes a backend and throw exception in `Engine` constructor if backend cannot be initialized. +- Provides information about devices enabled by a particular backend, e.g. how many devices, their properties and so on. +- Loads or imports [executable network](@ref executable_network) objects. + In addition to the Inference Engine Public API, the Inference Engine provides the Plugin API, which is a set of functions and helper classes that simplify new plugin development: - header files in the `inference_engine/src/plugin_api` directory @@ -18,8 +28,10 @@ Based on that, declaration of a plugin class can look as follows: #### Class Fields -The provided plugin class also has a single field: +The provided plugin class also has several fields: +* `_backend` - a backend engine that is used to perform actual computations for network inference. For `Template` plugin `ngraph::runtime::Backend` is used which performs computations using ngraph reference implementations. +* `_waitExecutor` - a task executor that waits for a response from a device about device tasks completion. * `_cfg` of type `Configuration`: @snippet src/template_config.hpp configuration:header @@ -28,6 +40,7 @@ As an example, a plugin configuration has three value parameters: - `deviceId` - particular device ID to work with. Applicable if a plugin supports more than one `Template` device. In this case, some plugin methods, like `SetConfig`, `QueryNetwork`, and `LoadNetwork`, must support the CONFIG_KEY(KEY_DEVICE_ID) parameter. - `perfCounts` - boolean value to identify whether to collect performance counters during [Inference Request](@ref infer_request) execution. +- `_streamsExecutorConfig` - configuration of `InferenceEngine::IStreamsExecutor` to handle settings of multi-threaded context. ### Engine Constructor @@ -47,25 +60,45 @@ A plugin must define a device name enabled via the `_pluginName` field of a base of the public InferenceEngine::InferencePluginInternal::LoadNetwork method that calls plugin-specific `LoadExeNetworkImpl`, which is defined in a derived class. This is the most important function of the `Plugin` class and creates an instance of compiled `ExecutableNetwork`, -which holds a hardware-dependent compiled graph in an internal representation: +which holds a backend-dependent compiled graph in an internal representation: @snippet src/template_plugin.cpp plugin:load_exe_network_impl Before a creation of an `ExecutableNetwork` instance via a constructor, a plugin may check if a provided InferenceEngine::ICNNNetwork object is supported by a device. In the example above, the plugin checks precision information. +The very important part before creation of `ExecutableNetwork` instance is to call `TransformNetwork` method which applies ngraph transformation passes. + Actual graph compilation is done in the `ExecutableNetwork` constructor. Refer to the [ExecutableNetwork Implementation Guide](@ref executable_network) for details. > **NOTE**: Actual configuration map used in `ExecutableNetwork` is constructed as a base plugin > configuration set via `Plugin::SetConfig`, where some values are overwritten with `config` passed to `Plugin::LoadExeNetworkImpl`. > Therefore, the config of `Plugin::LoadExeNetworkImpl` has a higher priority. +### `TransformNetwork()` + +The function accepts a const shared pointer to `ngraph::Function` object and performs the following steps: + +1. Deep copies a const object to a local object, which can later be modified. +2. Applies common and plugin-specific transformations on a copied graph to make the graph more friendly to hardware operations. For details how to write custom plugin-specific transformation, please, refer to [Writing ngraph transformations](@ref new_ngraph_transformation) guide. See detailed topics about network representation: + * [Intermediate Representation and Operation Sets](../_docs_MO_DG_IR_and_opsets.html) + * [Quantized networks](@ref quantized_networks). + +@snippet src/template_plugin.cpp plugin:transform_network + +> **NOTE**: After all these transformations, a `ngraph::Function` object cointains operations which can be perfectly mapped to backend kernels. E.g. if backend has kernel computing `A + B` operations at once, the `TransformNetwork` function should contain a pass which fuses operations `A` and `B` into a single custom operation `A + B` which fits backend kernels set. + ### `QueryNetwork()` Use the method with the `HETERO` mode, which allows to distribute network execution between different devices based on the `ngraph::Node::get_rt_info()` map, which can contain the `"affinity"` key. The `QueryNetwork` method analyzes operations of provided `network` and returns a list of supported -operations via the InferenceEngine::QueryNetworkResult structure: +operations via the InferenceEngine::QueryNetworkResult structure. The `QueryNetwork` firstly applies `TransformNetwork` passes to input `ngraph::Function` argument. After this, the transformed network in ideal case contains only operations are 1:1 mapped to kernels in computational backend. In this case, it's very easy to analyze which operations is supposed (`_backend` has a kernel for such operation or extensions for the operation is provided) and not supported (kernel is missed in `_backend`): + +1. Store original names of all operations in input `ngraph::Function` +2. Apply `TransformNetwork` passes. Note, the names of operations in a transformed network can be different and we need to restore the mapping in the steps below. +3. Construct `supported` and `unsupported` maps which contains names of original operations. Note, that since the inference is performed using ngraph reference backend, the decision whether the operation is supported or not depends on whether the latest OpenVINO opset contains such operation. +4. `QueryNetworkResult.supportedLayersMap` contains only operations which are fully supported by `_backend`. @snippet src/template_plugin.cpp plugin:query_network @@ -83,7 +116,7 @@ Sets new values for plugin configuration keys: @snippet src/template_plugin.cpp plugin:set_config In the snippet above, the `Configuration` class overrides previous configuration values with the new -ones. All these values are used during hardware-specific graph compilation and execution of inference requests. +ones. All these values are used during backend specific graph compilation and execution of inference requests. > **NOTE**: The function must throw an exception if it receives an unsupported configuration key. @@ -111,7 +144,7 @@ all devices of the same `Template` type with automatic logic of the `MULTI` devi in the `option` parameter as `{ CONFIG_KEY(KEY_DEVICE_ID), "deviceID" }`. - METRIC_KEY(SUPPORTED_METRICS) - list of metrics supported by a plugin - METRIC_KEY(SUPPORTED_CONFIG_KEYS) - list of configuration keys supported by a plugin that -affects their behavior during a hardware-specific graph compilation or an inference requests execution +affects their behavior during a backend specific graph compilation or an inference requests execution - METRIC_KEY(OPTIMIZATION_CAPABILITIES) - list of optimization capabilities of a device. For example, supported data types and special optimizations for them. - Any other device-specific metrics. In this case, place metrics declaration and possible values to @@ -128,9 +161,9 @@ The snippet below provides an example of the implementation for `GetMetric`: ### `ImportNetworkImpl()` -The importing network mechanism allows to import a previously exported hardware-specific graph and wrap it +The importing network mechanism allows to import a previously exported backend specific graph and wrap it using an [ExecutableNetwork](@ref executable_network) object. This functionality is useful if -hardware-specific graph compilation takes significant time and/or cannot be done on a target host +backend specific graph compilation takes significant time and/or cannot be done on a target host device due to other reasons. **Implementation details:** The base plugin class InferenceEngine::InferencePluginInternal implements InferenceEngine::InferencePluginInternal::ImportNetwork @@ -141,7 +174,7 @@ implementation and define an output blob structure up to its needs. This can be useful if a plugin exports a blob in a special format for integration with other frameworks where a common Inference Engine header from a base class implementation is not appropriate. -During export of hardware-specific graph using `ExecutableNetwork::Export`, a plugin may export any +During export of backend specific graph using `ExecutableNetwork::Export`, a plugin may export any type of information it needs to import a compiled graph properly and check its correctness. For example, the export information may include: @@ -150,7 +183,7 @@ For example, the export information may include: throw an exception if the `model` stream contains wrong data. For example, if devices have different capabilities and a graph compiled for a particular device cannot be used for another, such type of information must be stored and checked during the import. -- Compiled hardware-specific graph itself +- Compiled backend specific graph itself - Information about precisions and shapes set by the user @snippet src/template_plugin.cpp plugin:import_network_impl diff --git a/docs/IE_PLUGIN_DG/PluginTesting.md b/docs/IE_PLUGIN_DG/PluginTesting.md index a4c11ed9984..53bddf82320 100644 --- a/docs/IE_PLUGIN_DG/PluginTesting.md +++ b/docs/IE_PLUGIN_DG/PluginTesting.md @@ -1,40 +1,58 @@ # Plugin Testing {#plugin_testing} -Inference Engine (IE) tests infrastructure provides a predefined set of functional tests and utilities exported via the Inference -Engine developer package. They are used to verify a plugin using the Inference Engine public API. +Inference Engine (IE) tests infrastructure provides a predefined set of functional tests and utilities. They are used to verify a plugin using the Inference Engine public API. All the tests are written in the [Google Test C++ framework](https://github.com/google/googletest). +Inference Engine Plugin tests are included in the `IE::funcSharedTests` CMake target which is built within the OpenVINO repository +(see [Build Plugin Using CMake](@ref plugin_build) guide). This library contains tests definitions (the tests bodies) which can be parametrized and instantiated in plugins depending on whether a plugin supports a particular feature, specific sets of parameters for test on supported operation set and so on. + +Test definitions are splitted into tests class declaration (see `inference_engine/tests/functional/plugin/shared/include`) and tests class implementation (see `inference_engine/tests/functional/plugin/shared/src`) and include the following scopes of plugin conformance tests: + +1. **Behavior tests** (`behavior` sub-folder), which are a separate test group to check that a plugin satisfies basic Inference +Engine concepts: plugin creation, multiple executable networks support, multiple synchronous and asynchronous inference requests support, and so on. See the next section with details how to instantiate the tests definition class with plugin-specific parameters. + +2. **Single layer tests** (`single_layer_tests` sub-folder). This groups of tests checks that a particular single layer can be inferenced on a device. An example of test instantiation based on test definition from `IE::funcSharedTests` library: + + - From the declaration of convolution test class we can see that it's a parametrized GoogleTest based class with the `convLayerTestParamsSet` tuple of parameters: + + @snippet single_layer_tests/convolution.hpp test_convolution:definition + + - Based on that, define a set of parameters for `Template` plugin functional test instantiation: + + @snippet single_layer_tests/convolution.cpp test_convolution:declare_parameters + + - Instantiate the test itself using standard GoogleTest macro `INSTANTIATE_TEST_CASE_P`: + + @snippet single_layer_tests/convolution.cpp test_convolution:instantiate + +3. **Sub-graph tests** (`subgraph_tests` sub-folder). This group of tests is designed to tests small patterns or combination of layers. E.g. when a particular topology is being enabled in a plugin e.g. TF ResNet-50, there is no need to add the whole topology to test tests. In opposite way, a particular repetative subgraph or pattern can be extracted from `ResNet-50` and added to the tests. The instantiation of the sub-graph tests is done in the same way as for single layer tests. +> **Note**, such sub-graphs or patterns for sub-graph tests should be added to `IE::ngraphFunctions` library first (this library is a pre-defined set of small `ngraph::Function`) and re-used in sub-graph tests after. + +4. **HETERO tests** (`subgraph_tests` sub-folder) contains tests for `HETERO` scenario (manual or automatic affinities settings, tests for `QueryNetwork`). + +5. **Other tests**, which contain tests for other scenarios and has the following types of tests: + - Tests for execution graph + - Etc. + +To use these tests for your own plugin development, link the `IE::funcSharedTests` library to your test binary and instantiate required test cases with desired parameters values. + +> **NOTE**: A plugin may contain its own tests for use cases that are specific to hardware or need to be extensively tested. + To build test binaries together with other build artifacts, use the `make all` command. For details, see [Build Plugin Using CMake*](@ref plugin_build). -Inference Engine Plugin tests are included in the `funcSharedTests` CMake target which is built within the Deep Learning Deployment Toolkit (DLDT) repository -(see [Build Plugin Using CMake](@ref plugin_build) guide). +### Tests for plugin-specific ngraph transformations -Test definitions: +Please, refer to [Transformation testing](@ref new_ngraph_transformation) guide. -1. **Conformance tests**, which are a separate test group to check that a plugin satisfies basic Inference -Engine concepts: plugin creation, multiple executable networks support, multiple synchronous and asynchronous inference requests support, and so on. -2. **Other API tests**, which contain the following types of tests: - - Per-layer tests. Located in the `single_layer_tests`and `subgraph_tests` folders. - - Tests for integration with the `InferenceEngine::Core` class. Located in the the `ie_class` folder. - - Tests to check that IE common preprocessing works with your plugin. The `io_blob_tests` folder. - -To use these tests for your own plugin development, link the `funcSharedTests` library to your test binary and -instantiate required test cases with desired parameters values. - -> **NOTE**: A plugin may contain its own tests for use cases that are specific to hardware or need to be extensively -> tested. Depending on your device positioning, you can implement more specific tests for your device. Such tests can -> be defined both for conformance and other API tests groups within your own test binary. - -How to Extend Inference Engine Plugin Tests -======================== +### How to Extend Inference Engine Plugin Tests Inference Engine Plugin tests are open for contribution. -Add common test case definitions applicable for all plugins to the `funcSharedTests` target within the DLDT repository. Then, any other plugin supporting corresponding functionality can instantiate the new test. +Add common test case definitions applicable for all plugins to the `IE::funcSharedTests` target within the DLDT repository. Then, any other plugin supporting corresponding functionality can instantiate the new test. All Inference Engine per-layer tests check test layers functionality. They are developed using nGraph functions as input graphs used by tests. In this case, to test a new layer with layer tests, extend -the `ngraphFunctions` CMake target, which is also included in the Inference Engine Developer package, with a new nGraph function +the `IE::ngraphFunctions` library, which is also included in the Inference Engine Developer package, with a new nGraph function including the corresponding operation. -> **NOTE**: When implementing a new subgraph test, add new single-layer tests for each operation of the subgraph. \ No newline at end of file +> **NOTE**: When implementing a new subgraph test, add new single-layer tests for each operation of the subgraph if such test does not exist. \ No newline at end of file diff --git a/docs/template_plugin/CMakeLists.txt b/docs/template_plugin/CMakeLists.txt index df6c02e4de0..182c6d1a54b 100644 --- a/docs/template_plugin/CMakeLists.txt +++ b/docs/template_plugin/CMakeLists.txt @@ -25,7 +25,6 @@ if(ENABLE_TESTS) if(ENABLE_FUNCTIONAL_TESTS) add_subdirectory(tests/functional) endif() - endif() # [cmake:main] diff --git a/docs/template_plugin/README.md b/docs/template_plugin/README.md index 49d8cf0ec2e..d6128390d24 100644 --- a/docs/template_plugin/README.md +++ b/docs/template_plugin/README.md @@ -1,6 +1,7 @@ # template-plugin Template Plugin for Inference Engine which demonstrates basics of how Inference Engine plugin can be built and implemented on top of Inference Engine Developer Package and Plugin API. +As a backend for actual computations ngraph reference implementations is used, so the Template plugin is fully functional. ## How to build @@ -8,7 +9,7 @@ Template Plugin for Inference Engine which demonstrates basics of how Inference $ cd $DLDT_HOME $ mkdir $DLDT_HOME/build $ cd $DLDT_HOME/build -$ cmake -DENABLE_TESTS=ON -DENABLE_BEH_TESTS=ON -DENABLE_FUNCTIONAL_TESTS=ON .. +$ cmake -DENABLE_TESTS=ON -DENABLE_FUNCTIONAL_TESTS=ON .. $ make -j8 $ cd $TEMPLATE_PLUGIN_HOME $ mkdir $TEMPLATE_PLUGIN_HOME/build diff --git a/docs/template_plugin/src/CMakeLists.txt b/docs/template_plugin/src/CMakeLists.txt index f5921e20d97..86b0e9eb877 100644 --- a/docs/template_plugin/src/CMakeLists.txt +++ b/docs/template_plugin/src/CMakeLists.txt @@ -25,6 +25,7 @@ target_include_directories(${TARGET_NAME} PRIVATE target_include_directories(${TARGET_NAME} PRIVATE "${IE_MAIN_TEMPLATE_PLUGIN_SOURCE_DIR}/include") +# link common Inference Engine libraries target_link_libraries(${TARGET_NAME} PRIVATE IE::inference_engine IE::inference_engine_transformations diff --git a/docs/template_plugin/src/template_async_infer_request.cpp b/docs/template_plugin/src/template_async_infer_request.cpp index f1024185c44..fed650962c7 100644 --- a/docs/template_plugin/src/template_async_infer_request.cpp +++ b/docs/template_plugin/src/template_async_infer_request.cpp @@ -2,12 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include - -#include - #include "template_async_infer_request.hpp" -#include "template_executable_network.hpp" using namespace TemplatePlugin; @@ -19,11 +14,13 @@ TemplateAsyncInferRequest::TemplateAsyncInferRequest( const InferenceEngine::ITaskExecutor::Ptr& callbackExecutor) : AsyncInferRequestThreadSafeDefault(inferRequest, cpuTaskExecutor, callbackExecutor), _inferRequest(inferRequest), _waitExecutor(waitExecutor) { - constexpr const auto remoteDevice = false; - // By default single stage pipeline is created. + // In current implementation we have CPU only tasks and no needs in 2 executors + // So, by default single stage pipeline is created. // This stage executes InferRequest::Infer() using cpuTaskExecutor. // But if remote asynchronous device is used the pipeline can by splitted tasks that are executed by cpuTaskExecutor // and waiting tasks. Waiting tasks can lock execution thread so they use separate threads from other executor. + constexpr const auto remoteDevice = false; + if (remoteDevice) { _pipeline = { {cpuTaskExecutor, [this] { diff --git a/docs/template_plugin/src/template_config.cpp b/docs/template_plugin/src/template_config.cpp index b01f7a6cabd..7297c5effbd 100644 --- a/docs/template_plugin/src/template_config.cpp +++ b/docs/template_plugin/src/template_config.cpp @@ -2,15 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 // - -#include -#include -#include - -#include #include #include -#include #include #include "template_config.hpp" diff --git a/docs/template_plugin/src/template_config.hpp b/docs/template_plugin/src/template_config.hpp index 453c85a3e6d..b57d40a63b3 100644 --- a/docs/template_plugin/src/template_config.hpp +++ b/docs/template_plugin/src/template_config.hpp @@ -4,10 +4,8 @@ #pragma once -#include #include #include -#include #include @@ -15,9 +13,6 @@ namespace TemplatePlugin { -template -using IOMap = std::unordered_map; - // ! [configuration:header] using ConfigMap = std::map; diff --git a/docs/template_plugin/src/template_executable_network.cpp b/docs/template_plugin/src/template_executable_network.cpp index b0d0e66e80e..98b6e7dd9f7 100644 --- a/docs/template_plugin/src/template_executable_network.cpp +++ b/docs/template_plugin/src/template_executable_network.cpp @@ -2,19 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 // -#include -#include -#include -#include -#include -#include -#include - #include -#include #include #include -#include
#include "template/template_config.hpp" #include "template_plugin.hpp" @@ -23,19 +13,18 @@ using namespace TemplatePlugin; // ! [executable_network:ctor_cnnnetwork] -TemplatePlugin::ExecutableNetwork::ExecutableNetwork(const std::shared_ptr& function, - const Configuration& cfg, - const Plugin::Ptr& plugin) : +TemplatePlugin::ExecutableNetwork::ExecutableNetwork(const std::shared_ptr& function, + const Configuration& cfg, + const Plugin::Ptr& plugin) : InferenceEngine::ExecutableNetworkThreadSafeDefault(nullptr, nullptr), // Disable default threads creation _cfg(cfg), - _plugin(plugin), - _function(function) { + _plugin(plugin) { // TODO: if your plugin supports device ID (more that single instance of device can be on host machine) // you should select proper device based on KEY_DEVICE_ID or automatic behavior // In this case, _waitExecutor should also be created per device. try { - CompileGraph(); - InitExecutor(); + CompileNetwork(function); + InitExecutor(); // creates thread-based executor using for async requests } catch (const InferenceEngineException&) { throw; } catch (const std::exception & e) { @@ -53,12 +42,17 @@ TemplatePlugin::ExecutableNetwork::ExecutableNetwork(std::istream & _cfg(cfg), _plugin(plugin) { // TODO: since Import network is not a mandatory functionality, this ctor can just be removed + THROW_IE_EXCEPTION << NOT_IMPLEMENTED_str; } // ! [executable_network:ctor_import_stream] -// ! [executable_network:compile_graph] -void TemplatePlugin::ExecutableNetwork::CompileGraph() { - // TODO: perform actual graph compilation taking `_cfg` into account +// ! [executable_network:map_graph] +// forward declaration +std::shared_ptr TransformNetwork(const std::shared_ptr& function); + +void TemplatePlugin::ExecutableNetwork::CompileNetwork(const std::shared_ptr& function) { + // TODO: perform actual graph compilation / mapping to backend graph representation / kernels + _function = TransformNetwork(function); // Generate backend specific blob mappings. For example Inference Engine uses not ngraph::Result nodes friendly name // as inference request output names but the name of the layer before. @@ -74,13 +68,14 @@ void TemplatePlugin::ExecutableNetwork::CompileGraph() { _inputIndex.emplace(parameter->get_friendly_name(), _function->get_parameter_index(parameter)); } - // Perform any other steps like allocation and filling device buffers, and so on + // Perform any other steps like allocation and filling backend specific memory handles and so on } -// ! [executable_network:compile_graph] +// ! [executable_network:map_graph] + // ! [executable_network:init_executor] void TemplatePlugin::ExecutableNetwork::InitExecutor() { - // Default mutlitthreaded configuration is balanced for throughtput and latency cases and takes into account + // Default multi-threaded configuration is balanced for throughtput and latency cases and takes into account // real hardware cores and NUMA nodes. auto streamsExecutorConfig = InferenceEngine::IStreamsExecutor::Config::MakeDefaultMultiThreaded(_cfg._streamsExecutorConfig); streamsExecutorConfig._name = "TemplateStreamsExecutor"; @@ -151,7 +146,8 @@ void TemplatePlugin::ExecutableNetwork::GetMetric(const std::string &name, Infer // ! [executable_network:get_metric] // ! [executable_network:export_impl] -void TemplatePlugin::ExecutableNetwork::ExportImpl(std::ostream& dlaModel) { +void TemplatePlugin::ExecutableNetwork::ExportImpl(std::ostream& modelStream) { // TODO: Code which exports graph from std::ostream + THROW_IE_EXCEPTION << NOT_IMPLEMENTED_str; } // ! [executable_network:export_impl] diff --git a/docs/template_plugin/src/template_executable_network.hpp b/docs/template_plugin/src/template_executable_network.hpp index b2755b39c69..01bcc3270af 100644 --- a/docs/template_plugin/src/template_executable_network.hpp +++ b/docs/template_plugin/src/template_executable_network.hpp @@ -5,26 +5,14 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include +#include #include "template_config.hpp" #include "template_infer_request.hpp" #include "template_async_infer_request.hpp" +#include + namespace TemplatePlugin { class Plugin; @@ -36,9 +24,9 @@ class Plugin; // ! [executable_network:header] class ExecutableNetwork : public InferenceEngine::ExecutableNetworkThreadSafeDefault { public: - ExecutableNetwork(const std::shared_ptr& function, - const Configuration& cfg, - const std::shared_ptr& plugin); + ExecutableNetwork(const std::shared_ptr& function, + const Configuration& cfg, + const std::shared_ptr& plugin); ExecutableNetwork(std::istream& model, const Configuration& cfg, @@ -58,7 +46,7 @@ public: private: friend class TemplateInferRequest; - void CompileGraph(); + void CompileNetwork(const std::shared_ptr& function); void InitExecutor(); std::atomic _requestId = {0}; diff --git a/docs/template_plugin/src/template_function_transformation.cpp b/docs/template_plugin/src/template_function_transformation.cpp index 671c76d3a56..aa2299d779c 100644 --- a/docs/template_plugin/src/template_function_transformation.cpp +++ b/docs/template_plugin/src/template_function_transformation.cpp @@ -4,8 +4,6 @@ #include "template_function_transformation.hpp" -#include - using namespace ngraph; // ! [function_pass:template_transformation_cpp] diff --git a/docs/template_plugin/src/template_function_transformation.hpp b/docs/template_plugin/src/template_function_transformation.hpp index 398aa3f057a..72938bec81c 100644 --- a/docs/template_plugin/src/template_function_transformation.hpp +++ b/docs/template_plugin/src/template_function_transformation.hpp @@ -4,10 +4,7 @@ #pragma once -#include -#include - -#include +#include namespace ngraph { namespace pass { diff --git a/docs/template_plugin/src/template_infer_request.cpp b/docs/template_plugin/src/template_infer_request.cpp index e33e1c92927..3d15842f727 100644 --- a/docs/template_plugin/src/template_infer_request.cpp +++ b/docs/template_plugin/src/template_infer_request.cpp @@ -125,7 +125,7 @@ void TemplateInferRequest::InferImpl() { // TODO: fill with actual list of pipeline stages, which are executed synchronously for sync infer requests inferPreprocess(); startPipeline(); - waitPipeline(); + waitPipeline(); // does nothing in current implementation inferPostprocess(); } // ! [infer_request:infer_impl] @@ -208,49 +208,53 @@ void TemplateInferRequest::inferPreprocess() { } // ! [infer_request:infer_preprocess] +// ! [infer_request:start_pipeline] void TemplateInferRequest::startPipeline() { IE_PROFILING_AUTO_SCOPE_TASK(_profilingTask[StartPipeline]) auto start = Time::now(); _executable->call(_outputTensors, _inputTensors); _durations[StartPipeline] = Time::now() - start; } +// ! [infer_request:start_pipeline] void TemplateInferRequest::waitPipeline() { IE_PROFILING_AUTO_SCOPE_TASK(_profilingTask[WaitPipeline]) auto start = Time::now(); // TODO: Wait pipeline using driver API or other synchronizations methods + // NOTE: not used in current implementation since `startPipeline` executes pipiline synchronously _durations[WaitPipeline] = Time::now() - start; } +// ! [infer_request:infer_postprocess] void TemplateInferRequest::inferPostprocess() { IE_PROFILING_AUTO_SCOPE_TASK(_profilingTask[Postprocess]); auto start = Time::now(); for (auto&& output : _outputs) { auto outputBlob = output.second; auto networkOutput = _networkOutputBlobs[output.first]; + // perform precision conversion of network output's precision and computational + // graph output's precision are different if (outputBlob->getTensorDesc().getPrecision() != networkOutput->getTensorDesc().getPrecision()) { blobCopy(networkOutput, outputBlob); } } _durations[Postprocess] = Time::now() - start; } +// ! [infer_request:infer_postprocess] // ! [infer_request:get_performance_counts] void TemplateInferRequest::GetPerformanceCounts(std::map &perfMap) const { InferenceEngineProfileInfo info; info.execution_index = 0; info.status = InferenceEngineProfileInfo::EXECUTED; + info.cpu_uSec = info.realTime_uSec = _durations[Preprocess].count(); perfMap["1. input preprocessing"] = info; - info.cpu_uSec = 0; - info.realTime_uSec = 0; + info.cpu_uSec = info.realTime_uSec = 0; perfMap["2. input transfer to a device"] = info; - info.cpu_uSec = 0; - info.status = InferenceEngineProfileInfo::EXECUTED; info.cpu_uSec = info.realTime_uSec = _durations[StartPipeline].count(); perfMap["3. execution time"] = info; - info.cpu_uSec = 0; - info.realTime_uSec = 0; + info.cpu_uSec = info.realTime_uSec = 0; perfMap["4. output transfer from a device"] = info; info.cpu_uSec = info.realTime_uSec = _durations[Postprocess].count(); perfMap["5. output postprocessing"] = info; diff --git a/docs/template_plugin/src/template_infer_request.hpp b/docs/template_plugin/src/template_infer_request.hpp index ce42b250617..070965054b7 100644 --- a/docs/template_plugin/src/template_infer_request.hpp +++ b/docs/template_plugin/src/template_infer_request.hpp @@ -17,7 +17,6 @@ #include #include -#include #include #include @@ -47,8 +46,6 @@ public: void waitPipeline(); void inferPostprocess(); - std::shared_ptr _executableNetwork; - private: void allocateDeviceBuffers(); void allocateBlobs(); @@ -61,6 +58,7 @@ private: numOfStages }; + std::shared_ptr _executableNetwork; std::array _profilingTask; // for performance counters std::array, numOfStages> _durations; diff --git a/docs/template_plugin/src/template_pattern_transformation.cpp b/docs/template_plugin/src/template_pattern_transformation.cpp index 1e2060fbc7b..e8ca30c9cf5 100644 --- a/docs/template_plugin/src/template_pattern_transformation.cpp +++ b/docs/template_plugin/src/template_pattern_transformation.cpp @@ -5,9 +5,10 @@ #include "template_pattern_transformation.hpp" #include "template_function_transformation.hpp" -#include #include +#include #include +#include using namespace ngraph; diff --git a/docs/template_plugin/src/template_pattern_transformation.hpp b/docs/template_plugin/src/template_pattern_transformation.hpp index 0220a6d6b9b..c9346ef59e4 100644 --- a/docs/template_plugin/src/template_pattern_transformation.hpp +++ b/docs/template_plugin/src/template_pattern_transformation.hpp @@ -4,10 +4,7 @@ #pragma once -#include -#include - -#include +#include namespace ngraph { namespace pass { diff --git a/docs/template_plugin/src/template_plugin.cpp b/docs/template_plugin/src/template_plugin.cpp index 55113183166..4343c20f4a1 100644 --- a/docs/template_plugin/src/template_plugin.cpp +++ b/docs/template_plugin/src/template_plugin.cpp @@ -2,34 +2,20 @@ // SPDX-License-Identifier: Apache-2.0 // - -#include -#include -#include -#include -#include -#include -#include - #include -#include
#include -#include -#include -#include -#include -#include -#include -#include -#include -#include + #include -#include +#include +#include + +#include #include #include #include #include #include + #include "template/template_config.hpp" #include "template_plugin.hpp" #include "template_executable_network.hpp" @@ -40,10 +26,14 @@ using namespace TemplatePlugin; // ! [plugin:ctor] Plugin::Plugin() { - // TODO: fill with actual device name + // TODO: fill with actual device name, backend engine _pluginName = "TEMPLATE"; + + // create ngraph backend which performs inference using ngraph reference implementations ngraph::runtime::Backend::set_backend_shared_library_search_directory(""); _backend = ngraph::runtime::Backend::create("INTERPRETER"); + + // create default stream executor with a given name _waitExecutor = ExecutorManager::getInstance()->getIdleCPUStreamsExecutor({"TemplateWaitExecutor"}); } // ! [plugin:ctor] @@ -58,9 +48,10 @@ Plugin::~Plugin() { } // ! [plugin:dtor] -// ! [plugin:transform] -std::shared_ptr Plugin::Transform(const std::shared_ptr& function) { - // 1.Copy ngraph::Function first to apply some transformations which modify original ngraph::Function +// ! [plugin:transform_network] + +std::shared_ptr TransformNetwork(const std::shared_ptr& function) { + // 1. Copy ngraph::Function first to apply some transformations which modify original ngraph::Function const bool shareConsts = false, constFolding = false; std::vector<::ngraph::element::Type> new_types; std::vector<::ngraph::PartialShape> new_shapes; @@ -70,10 +61,11 @@ std::shared_ptr Plugin::Transform(const std::shared_ptrget_element_type()); } - auto copyFunction = ngraph::specialize_function(std::const_pointer_cast(function), + auto clonedNetwork = ngraph::specialize_function(std::const_pointer_cast(function), new_types, new_shapes, std::vector(new_types.size(), nullptr), constFolding, shareConsts); - copyFunction->set_friendly_name(function->get_friendly_name()); + auto transformedNetwork = clonedNetwork; + transformedNetwork->set_friendly_name(function->get_friendly_name()); // 2. Perform common optimizations and device-specific transformations ngraph::pass::Manager passManager; @@ -86,16 +78,12 @@ std::shared_ptr Plugin::Transform(const std::shared_ptrget_ordered_ops()) { - // TODO: map ngraph `op` to device operation - } - return copyFunction; + return transformedNetwork; } -// ! [plugin:transform] +// ! [plugin:transform_network] // ! [plugin:load_exe_network_impl] InferenceEngine::ExecutableNetworkInternal::Ptr Plugin::LoadExeNetworkImpl(const InferenceEngine::ICNNNetwork & network, @@ -135,7 +123,7 @@ InferenceEngine::ExecutableNetworkInternal::Ptr Plugin::LoadExeNetworkImpl(const THROW_IE_EXCEPTION << "TEMPLATE plugin can compile only IR v10 networks"; } - return std::make_shared(Transform(function), cfg, std::static_pointer_cast(shared_from_this())); + return std::make_shared(function, cfg, std::static_pointer_cast(shared_from_this())); } // ! [plugin:load_exe_network_impl] @@ -166,14 +154,17 @@ void Plugin::QueryNetwork(const ICNNNetwork &network, const ConfigMap& config, Q if (function == nullptr) { THROW_IE_EXCEPTION << "Template Plugin supports only ngraph cnn network representation"; } - // First of all we should store initial input operation set + + // 1. First of all we should store initial input operation set std::unordered_set originalOps; for (auto&& node : function->get_ops()) { originalOps.emplace(node->get_friendly_name()); } - // It is needed to apply all transformations as it is done in LoadExeNetworkImpl - auto transformedFunction = Transform(function); - // The same input node can be transformed into supported and unsupported backend node + + // 2. It is needed to apply all transformations as it is done in LoadExeNetworkImpl + auto transformedFunction = TransformNetwork(function); + + // 3. The same input node can be transformed into supported and unsupported backend node // So we need store as supported ether unsupported node sets std::unordered_set supported; std::unordered_set unsupported; @@ -183,6 +174,7 @@ void Plugin::QueryNetwork(const ICNNNetwork &network, const ConfigMap& config, Q // Extract transformation history from transformed node as list of nodes for (auto&& fusedLayerName : ngraph::getFusedNamesVector(node)) { // Filter just nodes from original operation set + // TODO: fill with actual decision rules based on whether kernel is supported by backend if (contains(originalOps, fusedLayerName)) { if (opset.contains_type_insensitive(fusedLayerName)) { supported.emplace(fusedLayerName); @@ -193,7 +185,8 @@ void Plugin::QueryNetwork(const ICNNNetwork &network, const ConfigMap& config, Q } } } - // The result set should contains just nodes from supported set + + // 4. The result set should contains just nodes from supported set for (auto&& layerName : supported) { if (!contains(unsupported, layerName)) { res.supportedLayersMap.emplace(layerName, GetName()); @@ -205,6 +198,7 @@ void Plugin::QueryNetwork(const ICNNNetwork &network, const ConfigMap& config, Q // ! [plugin:add_extension] void Plugin::AddExtension(InferenceEngine::IExtensionPtr /*extension*/) { // TODO: add extensions if plugin supports extensions + THROW_IE_EXCEPTION << NOT_IMPLEMENTED_str; } // ! [plugin:add_extension] diff --git a/docs/template_plugin/src/template_plugin.hpp b/docs/template_plugin/src/template_plugin.hpp index 6e611e58ed3..22519dfe55b 100644 --- a/docs/template_plugin/src/template_plugin.hpp +++ b/docs/template_plugin/src/template_plugin.hpp @@ -4,18 +4,11 @@ #pragma once -#include -#include +#include "template_config.hpp" +#include "template_executable_network.hpp" #include -#include -#include -#include -#include -#include - -#include "template_executable_network.hpp" -#include "template_config.hpp" +#include "backend.hpp" #include "backend.hpp" @@ -45,10 +38,8 @@ private: friend class ExecutableNetwork; friend class TemplateInferRequest; - static std::shared_ptr Transform(const std::shared_ptr& function); - - Configuration _cfg; std::shared_ptr _backend; + Configuration _cfg; InferenceEngine::ITaskExecutor::Ptr _waitExecutor; }; diff --git a/docs/template_plugin/tests/functional/CMakeLists.txt b/docs/template_plugin/tests/functional/CMakeLists.txt index be10445bf5d..627e112c4e2 100644 --- a/docs/template_plugin/tests/functional/CMakeLists.txt +++ b/docs/template_plugin/tests/functional/CMakeLists.txt @@ -3,7 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 # -set(TARGET_NAME TemplateFuncTests) +# [cmake:functional_tests] +set(TARGET_NAME templateFuncTests) addIeTargetTest( NAME ${TARGET_NAME} @@ -12,9 +13,10 @@ addIeTargetTest( templatePlugin LINK_LIBRARIES IE::funcSharedTests + INCLUDES + "${IE_MAIN_TEMPLATE_PLUGIN_SOURCE_DIR}/include" ADD_CPPLINT LABELS TEMPLATE ) - -target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../include) +# [cmake:functional_tests] diff --git a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/config.cpp b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/config.cpp index 6b9d7ebf1e9..981e0527a9b 100644 --- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/config.cpp +++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/config.cpp @@ -8,49 +8,51 @@ #include