Update md files. Add cpp in docs/examples (#1769)
* Update md files. Add cpp in docs/examples * Normalize all the line endings * Fix block_id in snippets * Fix utf-8 encoding * Add new folder for snippets * Fix issues with compiling code from snippets * Added conteiner iterator fix
This commit is contained in:
Polina Mishanina
@@ -20,10 +20,7 @@ There are two ways to check if CPU device can support bfloat16 computations for
|
||||
1. Query the instruction set via system `lscpu | grep avx512_bf16` or `cat /proc/cpuinfo | grep avx512_bf16`.
|
||||
2. Use [Query API](InferenceEngine_QueryAPI.md) with `METRIC_KEY(OPTIMIZATION_CAPABILITIES)`, which should return `BF16` in the list of CPU optimization options:
|
||||
|
||||
```cpp
|
||||
InferenceEngine::Core core;
|
||||
auto cpuOptimizationCapabilities = core.GetMetric("CPU", METRIC_KEY(OPTIMIZATION_CAPABILITIES)).as<std::vector<std::string>>();
|
||||
```
|
||||
@snippet openvino/docs/snippets/Bfloat16Inference0.cpp part0
|
||||
|
||||
Current Inference Engine solution for bfloat16 inference uses Intel® Math Kernel Library for Deep Neural Networks (Intel® MKL-DNN) and supports inference of the following layers in BF16 computation mode:
|
||||
* Convolution
|
||||
@@ -49,18 +46,11 @@ Bfloat16 data usage provides the following benefits that increase performance:
|
||||
For default optimization on CPU, source model converts from FP32 or FP16 to BF16 and executes internally on platforms with native BF16 support. In that case, `KEY_ENFORCE_BF16` is set to `YES`.
|
||||
The code below demonstrates how to check if the key is set:
|
||||
|
||||
```cpp
|
||||
InferenceEngine::Core core;
|
||||
auto exeNetwork = core.LoadNetwork(network, "CPU");
|
||||
auto enforceBF16 = exeNetwork.GetConfig(PluginConfigParams::KEY_ENFORCE_BF16).as<std::string>();
|
||||
```
|
||||
@snippet openvino/docs/snippets/Bfloat16Inference1.cpp part1
|
||||
|
||||
To disable BF16 internal transformations, set the `KEY_ENFORCE_BF16` to `NO`. In this case, the model infers AS IS without modifications with precisions that were set on each layer edge.
|
||||
|
||||
```cpp
|
||||
InferenceEngine::Core core;
|
||||
core.SetConfig({ { CONFIG_KEY(ENFORCE_BF16), CONFIG_VALUE(NO) } }, "CPU");
|
||||
```
|
||||
@snippet openvino/docs/snippets/Bfloat16Inference2.cpp part2
|
||||
|
||||
An exception with message `Platform doesn't support BF16 format` is formed in case of setting `KEY_ENFORCE_BF16` to `YES` on CPU without native BF16 support.
|
||||
|
||||
|
||||
@@ -17,39 +17,8 @@ dynamically in all of its infer requests using <code>SetBatch()</code> method.
|
||||
The batch size that was set in passed <code>CNNNetwork</code> object will be used as a maximum batch size limit.
|
||||
|
||||
Here is a code example:
|
||||
```cpp
|
||||
int dynBatchLimit = FLAGS_bl; //take dynamic batch limit from command line option
|
||||
|
||||
// Read network model
|
||||
Core core;
|
||||
CNNNetwork network = core.ReadNetwork(modelFileName, weightFileName);
|
||||
|
||||
// enable dynamic batching and prepare for setting max batch limit
|
||||
const std::map<std::string, std::string> dyn_config =
|
||||
{ { PluginConfigParams::KEY_DYN_BATCH_ENABLED, PluginConfigParams::YES } };
|
||||
network.setBatchSize(dynBatchLimit);
|
||||
|
||||
// create executable network and infer request
|
||||
auto executable_network = core.LoadNetwork(network, "CPU", dyn_config);
|
||||
auto infer_request = executable_network.CreateInferRequest();
|
||||
|
||||
|
||||
...
|
||||
|
||||
|
||||
// process a set of images
|
||||
// dynamically set batch size for subsequent Infer() calls of this request
|
||||
size_t batchSize = imagesData.size();
|
||||
infer_request.SetBatch(batchSize);
|
||||
infer_request.Infer();
|
||||
|
||||
...
|
||||
|
||||
// process another set of images
|
||||
batchSize = imagesData2.size();
|
||||
infer_request.SetBatch(batchSize);
|
||||
infer_request.Infer();
|
||||
```
|
||||
@snippet openvino/docs/snippets/DynamicBatching.cpp part0
|
||||
|
||||
|
||||
## Limitations
|
||||
|
||||
@@ -64,11 +64,6 @@ InferenceEngine::IExtension::getImplementation returns the kernel implementation
|
||||
## Load Extension with Executable Kernels to Plugin
|
||||
|
||||
Use the `AddExtension` method of the general plugin interface to load your primitives:
|
||||
```cpp
|
||||
InferenceEngine::Core core;
|
||||
// Load CPU extension as a shared library
|
||||
auto extension_ptr = make_so_pointer<InferenceEngine::IExtension>("<shared lib path>");
|
||||
// Add extension to the CPU device
|
||||
core.AddExtension(extension_ptr, "CPU");
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/CPU_Kernel.cpp part0
|
||||
|
||||
|
||||
@@ -6,11 +6,8 @@ There are two options of using custom layer configuration file:
|
||||
|
||||
* Include a section with your kernels into the global automatically-loaded `cldnn_global_custom_kernels/cldnn_global_custom_kernels.xml` file, which is hosted in the `<INSTALL_DIR>/deployment_tools/inference_engine/bin/intel64/{Debug/Release}` folder
|
||||
* Call the `InferenceEngine::Core::SetConfig()` method from your application with the `InferenceEngine::PluginConfigParams::KEY_CONFIG_FILE` key and the configuration file name as a value before loading the network that uses custom layers to the plugin:
|
||||
```cpp
|
||||
InferenceEngine::Core core;
|
||||
// Load GPU Extensions
|
||||
core.SetConfig({ { InferenceEngine::PluginConfigParams::KEY_CONFIG_FILE, "<path_to_the_xml_file>" } }, "GPU");
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/GPU_Kernel.cpp part0
|
||||
|
||||
All Inference Engine samples, except trivial `hello_classification`,
|
||||
feature a dedicated command-line option `-c` to load custom kernels. For example, to load custom layers for the classification sample, run the command below:
|
||||
@@ -229,9 +226,9 @@ the values set by the Inference Engine, such as tensor sizes,
|
||||
floating-point, and integer kernel parameters. To get the dump, add the
|
||||
following line to your code that configures the GPU plugin to output the
|
||||
custom kernels:
|
||||
```cpp
|
||||
core.SetConfig({ { PluginConfigParams::KEY_DUMP_KERNELS, PluginConfigParams::YES } }, "GPU");
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/GPU_Kernel.cpp part1
|
||||
|
||||
When the Inference Engine compiles the kernels for the specific network,
|
||||
it also outputs the resulting code for the custom kernels. In the
|
||||
directory of your executable, find files like
|
||||
|
||||
@@ -29,12 +29,9 @@ File with tuned data is the result of this step.
|
||||
> **NOTE** If a filename passed to `KEY_TUNING_FILE` points to existing tuned data and you are tuning a new model, then this file will be extended by new data. This allows you to extend existing `cache.json` provided in the OpenVINO™ release package.
|
||||
|
||||
The example below shows how to set and use the key files:
|
||||
```cpp
|
||||
Core ie;
|
||||
ie.SetConfig({{ CONFIG_KEY(TUNING_MODE), CONFIG_VALUE(TUNING_CREATE) }}, "GPU");
|
||||
ie.SetConfig({{ CONFIG_KEY(TUNING_FILE), "/path/to/tuning/file.json" }}, "GPU");
|
||||
// Further LoadNetwork calls will use the specified tuning parameters
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/GPU_Kernels_Tuning.cpp part0
|
||||
|
||||
---
|
||||
|
||||
You can activate the inference with tuned data by setting `KEY_TUNING_MODE` flag to `TUNING_USE_EXISTING` and
|
||||
|
||||
@@ -10,24 +10,16 @@ the `-DNGRAPH_DEBUG_ENABLE=ON` option.
|
||||
|
||||
To visualize the nGraph function to the xDot format or to an image file, use the
|
||||
`ngraph::pass::VisualizeTree` graph transformation pass:
|
||||
```cpp
|
||||
#include <ngraph/pass/visualize_tree.hpp>
|
||||
|
||||
std::shared_ptr<ngraph::Function> nGraph;
|
||||
...
|
||||
ngraph::pass::VisualizeTree("after.png").run_on_function(nGraph); // Visualize the nGraph function to an image
|
||||
```
|
||||
@snippet openvino/docs/snippets/Graph_debug_capabilities0.cpp part0
|
||||
|
||||
## CNNNetwork
|
||||
|
||||
To serialize the CNNNetwork to the Inference Engine Intermediate Representation (IR) format, use the
|
||||
`CNNNetwork::serialize(...)` method:
|
||||
```cpp
|
||||
std::shared_ptr<ngraph::Function> nGraph;
|
||||
...
|
||||
CNNNetwork network(nGraph);
|
||||
network.serialize("test_ir.xml", "test_ir.bin");
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Graph_debug_capabilities1.cpp part1
|
||||
|
||||
> **NOTE**: CNNNetwork created from the nGraph function might differ from the original nGraph
|
||||
> function because the Inference Engine applies some graph transformation.
|
||||
|
||||
|
||||
@@ -23,10 +23,7 @@ The `InferenceEngine::ExecutableNetwork` class is also extended to support the Q
|
||||
|
||||
### GetAvailableDevices
|
||||
|
||||
```cpp
|
||||
InferenceEngine::Core core;
|
||||
std::vector<std::string> availableDevices = ie.GetAvailableDevices();
|
||||
```
|
||||
@snippet openvino/docs/snippets/InferenceEngine_QueryAPI0.cpp part0
|
||||
|
||||
The function returns list of available devices, for example:
|
||||
```
|
||||
@@ -49,10 +46,7 @@ Each device name can then be passed to:
|
||||
|
||||
The code below demonstrates how to understand whether `HETERO` device dumps `.dot` files with split graphs during the split stage:
|
||||
|
||||
```cpp
|
||||
InferenceEngine::Core core;
|
||||
bool dumpDotFile = core.GetConfig("HETERO", HETERO_CONFIG_KEY(DUMP_GRAPH_DOT)).as<bool>();
|
||||
```
|
||||
@snippet openvino/docs/snippets/InferenceEngine_QueryAPI1.cpp part1
|
||||
|
||||
For documentation about common configuration keys, refer to `ie_plugin_config.hpp`. Device specific configuration keys can be found in corresponding plugin folders.
|
||||
|
||||
@@ -60,10 +54,7 @@ For documentation about common configuration keys, refer to `ie_plugin_config.hp
|
||||
|
||||
* To extract device properties such as available device, device name, supported configuration keys, and others, use the `InferenceEngine::Core::GetMetric` method:
|
||||
|
||||
```cpp
|
||||
InferenceEngine::Core core;
|
||||
std::string cpuDeviceName = core.GetMetric("GPU", METRIC_KEY(FULL_DEVICE_NAME)).as<std::string>();
|
||||
```
|
||||
@snippet openvino/docs/snippets/InferenceEngine_QueryAPI2.cpp part2
|
||||
|
||||
A returned value looks as follows: `Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz`.
|
||||
|
||||
@@ -74,28 +65,18 @@ A returned value looks as follows: `Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz`.
|
||||
### GetMetric()
|
||||
|
||||
The method is used to get executable network specific metric such as `METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)`:
|
||||
```cpp
|
||||
InferenceEngine::Core core;
|
||||
auto exeNetwork = core.LoadNetwork(network, "CPU");
|
||||
auto nireq = exeNetwork.GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<unsigned int>();
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/InferenceEngine_QueryAPI3.cpp part3
|
||||
|
||||
Or the current temperature of `MYRIAD` device:
|
||||
```cpp
|
||||
InferenceEngine::Core core;
|
||||
auto exeNetwork = core.LoadNetwork(network, "MYRIAD");
|
||||
float temperature = exeNetwork.GetMetric(METRIC_KEY(DEVICE_THERMAL)).as<float>();
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/InferenceEngine_QueryAPI4.cpp part4
|
||||
|
||||
### GetConfig()
|
||||
|
||||
The method is used to get information about configuration values the executable network has been created with:
|
||||
|
||||
```cpp
|
||||
InferenceEngine::Core core;
|
||||
auto exeNetwork = core.LoadNetwork(network, "CPU");
|
||||
auto ncores = exeNetwork.GetConfig(PluginConfigParams::KEY_CPU_THREADS_NUM).as<std::string>();
|
||||
```
|
||||
@snippet openvino/docs/snippets/InferenceEngine_QueryAPI5.cpp part5
|
||||
|
||||
### SetConfig()
|
||||
|
||||
|
||||
@@ -28,27 +28,22 @@ Integration process includes the following steps:
|
||||
![integration_process]
|
||||
|
||||
1) **Create Inference Engine Core** to manage available devices and read network objects:
|
||||
```cpp
|
||||
InferenceEngine::Core core;
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Integrate_with_customer_application_new_API.cpp part0
|
||||
|
||||
2) **Read a model IR** created by the Model Optimizer (.xml is supported format):
|
||||
```cpp
|
||||
auto network = core.ReadNetwork("Model.xml");
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Integrate_with_customer_application_new_API.cpp part1
|
||||
|
||||
**Or read the model from ONNX format** (.onnx and .prototxt are supported formats). You can find more information about the ONNX format support in the document [ONNX format support in the OpenVINO™](./ONNX_Support.md).
|
||||
```cpp
|
||||
auto network = core.ReadNetwork("model.onnx");
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Integrate_with_customer_application_new_API.cpp part2
|
||||
|
||||
3) **Configure input and output**. Request input and output information using `InferenceEngine::CNNNetwork::getInputsInfo()`, and `InferenceEngine::CNNNetwork::getOutputsInfo()`
|
||||
methods:
|
||||
```cpp
|
||||
/** Take information about all topology inputs **/
|
||||
InferenceEngine::InputsDataMap input_info = network.getInputsInfo();
|
||||
/** Take information about all topology outputs **/
|
||||
InferenceEngine::OutputsDataMap output_info = network.getOutputsInfo();
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Integrate_with_customer_application_new_API.cpp part3
|
||||
|
||||
Optionally, set the number format (precision) and memory layout for inputs and outputs. Refer to the
|
||||
[Supported configurations](supported_plugins/Supported_Devices.md) chapter to choose the relevant configuration.
|
||||
|
||||
@@ -71,22 +66,8 @@ InferenceEngine::OutputsDataMap output_info = network.getOutputsInfo();
|
||||
> **NOTE**: Batch pre-processing is not supported if input color format is set to `ColorFormat::NV12`.
|
||||
|
||||
You can use the following code snippet to configure input and output:
|
||||
```cpp
|
||||
/** Iterate over all input info**/
|
||||
for (auto &item : input_info) {
|
||||
auto input_data = item.second;
|
||||
input_data->setPrecision(Precision::U8);
|
||||
input_data->setLayout(Layout::NCHW);
|
||||
input_data->getPreProcess().setResizeAlgorithm(RESIZE_BILINEAR);
|
||||
input_data->getPreProcess().setColorFormat(ColorFormat::RGB);
|
||||
}
|
||||
/** Iterate over all output info**/
|
||||
for (auto &item : output_info) {
|
||||
auto output_data = item.second;
|
||||
output_data->setPrecision(Precision::FP32);
|
||||
output_data->setLayout(Layout::NC);
|
||||
}
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Integrate_with_customer_application_new_API.cpp part4
|
||||
|
||||
> **NOTE**: NV12 input color format pre-processing differs from other color conversions. In case of NV12,
|
||||
> Inference Engine expects two separate image planes (Y and UV). You must use a specific
|
||||
@@ -109,45 +90,33 @@ for (auto &item : output_info) {
|
||||
|Layout | NCDHW | NCHW | CHW | NC | C |
|
||||
|
||||
4) **Load the model** to the device using `InferenceEngine::Core::LoadNetwork()`:
|
||||
```cpp
|
||||
auto executable_network = core.LoadNetwork(network, "CPU");
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Integrate_with_customer_application_new_API.cpp part5
|
||||
|
||||
It creates an executable network from a network object. The executable network is associated with single hardware device.
|
||||
It is possible to create as many networks as needed and to use them simultaneously (up to the limitation of the hardware resources).
|
||||
Third parameter is a configuration for plugin. It is map of pairs: (parameter name, parameter value). Choose device from
|
||||
[Supported devices](supported_plugins/Supported_Devices.md) page for more details about supported configuration parameters.
|
||||
```cpp
|
||||
/** Optional config. E.g. this enables profiling of performance counters. **/
|
||||
std::map<std::string, std::string> config = {{ PluginConfigParams::KEY_PERF_COUNT, PluginConfigParams::YES }};
|
||||
auto executable_network = core.LoadNetwork(network, "CPU", config);
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Integrate_with_customer_application_new_API.cpp part6
|
||||
|
||||
5) **Create an infer request**:
|
||||
```cpp
|
||||
auto infer_request = executable_network.CreateInferRequest();
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Integrate_with_customer_application_new_API.cpp part7
|
||||
|
||||
6) **Prepare input**. You can use one of the following options to prepare input:
|
||||
* **Optimal way for a single network.** Get blobs allocated by an infer request using `InferenceEngine::InferRequest::GetBlob()`
|
||||
and feed an image and the input data to the blobs. In this case, input data must be aligned (resized manually) with a
|
||||
given blob size and have a correct color format.
|
||||
```cpp
|
||||
/** Iterate over all input blobs **/
|
||||
for (auto & item : inputInfo) {
|
||||
auto input_name = item->first;
|
||||
/** Get input blob **/
|
||||
auto input = infer_request.GetBlob(input_name);
|
||||
/** Fill input tensor with planes. First b channel, then g and r channels **/
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Integrate_with_customer_application_new_API.cpp part8
|
||||
|
||||
* **Optimal way for a cascade of networks (output of one network is input for another).** Get output blob from the first
|
||||
request using `InferenceEngine::InferRequest::GetBlob()` and set it as input for the second request using
|
||||
`InferenceEngine::InferRequest::SetBlob()`.
|
||||
```cpp
|
||||
auto output = infer_request1->GetBlob(output_name);
|
||||
infer_request2->SetBlob(input_name, output);
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Integrate_with_customer_application_new_API.cpp part9
|
||||
|
||||
* **Optimal way to handle ROI (a ROI object located inside of input of one network is input for another).** It is
|
||||
possible to re-use shared input by several networks. You do not need to allocate separate input blob for a network if
|
||||
it processes a ROI object located inside of already allocated input of a previous network. For instance, when first
|
||||
@@ -156,38 +125,17 @@ infer_request2->SetBlob(input_name, output);
|
||||
In this case, it is allowed to re-use pre-allocated input blob (used by first network) by second network and just crop
|
||||
ROI without allocation of new memory using `InferenceEngine::make_shared_blob()` with passing of
|
||||
`InferenceEngine::Blob::Ptr` and `InferenceEngine::ROI` as parameters.
|
||||
```cpp
|
||||
/** inputBlob points to input of a previous network and
|
||||
cropROI contains coordinates of output bounding box **/
|
||||
InferenceEngine::Blob::Ptr inputBlob;
|
||||
InferenceEngine::ROI cropRoi;
|
||||
...
|
||||
|
||||
/** roiBlob uses shared memory of inputBlob and describes cropROI
|
||||
according to its coordinates **/
|
||||
auto roiBlob = InferenceEngine::make_shared_blob(inputBlob, cropRoi);
|
||||
infer_request2->SetBlob(input_name, roiBlob);
|
||||
```
|
||||
@snippet openvino/docs/snippets/Integrate_with_customer_application_new_API.cpp part10
|
||||
|
||||
Make sure that shared input is kept valid during execution of each network. Otherwise, ROI blob may be corrupted if the
|
||||
original input blob (that ROI is cropped from) has already been rewritten.
|
||||
|
||||
* Allocate input blobs of the appropriate types and sizes, feed an image and the input data to the blobs, and call
|
||||
`InferenceEngine::InferRequest::SetBlob()` to set these blobs for an infer request:
|
||||
```cpp
|
||||
/** Iterate over all input blobs **/
|
||||
for (auto & item : inputInfo) {
|
||||
auto input_data = item->second;
|
||||
/** Create input blob **/
|
||||
InferenceEngine::TBlob<unsigned char>::Ptr input;
|
||||
// assuming input precision was asked to be U8 in prev step
|
||||
input = InferenceEngine::make_shared_blob<unsigned char, InferenceEngine::SizeVector>(InferenceEngine::Precision:U8, input_data->getDims());
|
||||
input->allocate();
|
||||
infer_request->SetBlob(item.first, input);
|
||||
|
||||
/** Fill input tensor with planes. First b channel, then g and r channels **/
|
||||
...
|
||||
}
|
||||
```
|
||||
@snippet openvino/docs/snippets/Integrate_with_customer_application_new_API.cpp part11
|
||||
|
||||
A blob can be filled before and after `SetBlob()`.
|
||||
|
||||
> **NOTE:**
|
||||
@@ -208,15 +156,13 @@ for (auto & item : inputInfo) {
|
||||
|
||||
7) **Do inference** by calling the `InferenceEngine::InferRequest::StartAsync` and `InferenceEngine::InferRequest::Wait`
|
||||
methods for asynchronous request:
|
||||
```cpp
|
||||
infer_request->StartAsync();
|
||||
infer_request.Wait(IInferRequest::WaitMode::RESULT_READY);
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Integrate_with_customer_application_new_API.cpp part12
|
||||
|
||||
or by calling the `InferenceEngine::InferRequest::Infer` method for synchronous request:
|
||||
```cpp
|
||||
sync_infer_request->Infer();
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Integrate_with_customer_application_new_API.cpp part13
|
||||
|
||||
`StartAsync` returns immediately and starts inference without blocking main thread, `Infer` blocks
|
||||
main thread and returns when inference is completed.
|
||||
Call `Wait` for waiting result to become available for asynchronous request.
|
||||
@@ -238,17 +184,8 @@ exception.
|
||||
8) Go over the output blobs and **process the results**.
|
||||
Note that casting `Blob` to `TBlob` via `std::dynamic_pointer_cast` is not recommended way,
|
||||
better to access data via `buffer()` and `as()` methods as follows:
|
||||
```cpp
|
||||
for (auto &item : output_info) {
|
||||
auto output_name = item.first;
|
||||
auto output = infer_request.GetBlob(output_name);
|
||||
{
|
||||
auto const memLocker = output->cbuffer(); // use const memory locker
|
||||
// output_buffer is valid as long as the lifetime of memLocker
|
||||
const float *output_buffer = memLocker.as<const float *>();
|
||||
/** output_buffer[] - accessing output blob data **/
|
||||
|
||||
```
|
||||
@snippet openvino/docs/snippets/Integrate_with_customer_application_new_API.cpp part14
|
||||
|
||||
## Build Your Application
|
||||
|
||||
|
||||
@@ -26,52 +26,45 @@ The main responsibility of the `InferenceEngine::Core` class is to hide plugin s
|
||||
Common migration process includes the following steps:
|
||||
|
||||
1. Migrate from the `InferenceEngine::InferencePlugin` initialization:
|
||||
```cpp
|
||||
InferenceEngine::InferencePlugin plugin = InferenceEngine::PluginDispatcher({ FLAGS_pp }).getPluginByDevice(FLAGS_d);
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Migration_CoreAPI.cpp part0
|
||||
|
||||
to the `InferenceEngine::Core` class initialization:
|
||||
```cpp
|
||||
InferenceEngine::Core core;
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Migration_CoreAPI.cpp part1
|
||||
|
||||
2. Instead of using `InferenceEngine::CNNNetReader` to read IR:
|
||||
```cpp
|
||||
CNNNetReader network_reader;
|
||||
network_reader.ReadNetwork(fileNameToString(input_model));
|
||||
network_reader.ReadWeights(fileNameToString(input_model).substr(0, input_model.size() - 4) + ".bin");
|
||||
CNNNetwork network = network_reader.getNetwork();
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Migration_CoreAPI.cpp part2
|
||||
|
||||
read networks using the Core class:
|
||||
```cpp
|
||||
CNNNetwork network = core.ReadNetwork(input_model);
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Migration_CoreAPI.cpp part3
|
||||
|
||||
The Core class also allows reading models from the ONNX format (more information is [here](./ONNX_Support.md)):
|
||||
```cpp
|
||||
CNNNetwork network = core.ReadNetwork("model.onnx");
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Migration_CoreAPI.cpp part4
|
||||
|
||||
3. Instead of adding CPU device extensions to the plugin:
|
||||
```cpp
|
||||
plugin.AddExtension(std::make_shared<Extensions::Cpu::CpuExtensions>());
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Migration_CoreAPI.cpp part5
|
||||
|
||||
add extensions to CPU device using the Core class:
|
||||
```cpp
|
||||
core.AddExtension(std::make_shared<Extensions::Cpu::CpuExtensions>(), "CPU");
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Migration_CoreAPI.cpp part6
|
||||
|
||||
4. Instead of setting configuration keys to a particular plugin, set (key, value) pairs via `InferenceEngine::Core::SetConfig`
|
||||
```cpp
|
||||
core.SetConfig({{PluginConfigParams::KEY_CONFIG_FILE, FLAGS_c}}, "GPU");
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Migration_CoreAPI.cpp part7
|
||||
|
||||
> **NOTE**: If `deviceName` is omitted as the last argument, configuration is set for all Inference Engine devices.
|
||||
|
||||
5. Migrate from loading the network to a particular plugin:
|
||||
```cpp
|
||||
auto execNetwork = plugin.LoadNetwork(network, { });
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Migration_CoreAPI.cpp part8
|
||||
|
||||
to `InferenceEngine::Core::LoadNetwork` to a particular device:
|
||||
```cpp
|
||||
auto execNetwork = core.LoadNetwork(network, deviceName, { });
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/Migration_CoreAPI.cpp part9
|
||||
|
||||
After you have an instance of `InferenceEngine::ExecutableNetwork`, all other steps are as usual.
|
||||
|
||||
@@ -17,16 +17,9 @@ Two categories of API functions:
|
||||
|
||||
To list all supported ONNX ops in a specific version and domain, use the `get_supported_operators`
|
||||
as shown in the example below:
|
||||
```cpp
|
||||
const std::int64_t version = 12;
|
||||
const std::string domain = "ai.onnx";
|
||||
const std::set<std::string> supported_ops = ngraph::onnx_import::get_supported_operators(version, domain);
|
||||
|
||||
for(const auto& op : supported_ops)
|
||||
{
|
||||
std::cout << op << std::endl;
|
||||
}
|
||||
```
|
||||
@snippet openvino/docs/snippets/OnnxImporterTutorial0.cpp part0
|
||||
|
||||
The above code produces a list of all the supported operators for the `version` and `domain` you specified and outputs a list similar to this:
|
||||
```cpp
|
||||
Abs
|
||||
@@ -36,14 +29,8 @@ Xor
|
||||
```
|
||||
|
||||
To determine whether a specific ONNX operator in a particular version and domain is supported by the importer, use the `is_operator_supported` function as shown in the example below:
|
||||
```cpp
|
||||
const std::string op_name = "Abs";
|
||||
const std::int64_t version = 12;
|
||||
const std::string domain = "ai.onnx";
|
||||
const bool is_abs_op_supported = ngraph::onnx_import::is_operator_supported(op_name, version, domain);
|
||||
|
||||
std::cout << "Abs in version 12, domain `ai.onnx`is supported: " << (is_abs_op_supported ? "true" : "false") << std::endl;
|
||||
```
|
||||
@snippet openvino/docs/snippets/OnnxImporterTutorial1.cpp part1
|
||||
|
||||
## Import ONNX Model
|
||||
|
||||
@@ -68,33 +55,13 @@ As it was shown in [Build a Model with nGraph Library](nGraphTutorial.md), `std:
|
||||
|
||||
The code below shows how to convert the ONNX ResNet50 model to the nGraph function using `import_onnx_model` with the stream as an input:
|
||||
|
||||
```cpp
|
||||
const std::string resnet50_path = "resnet50/model.onnx";
|
||||
std::ifstream resnet50_stream(resnet50_path);
|
||||
if(resnet50_stream.is_open())
|
||||
{
|
||||
try
|
||||
{
|
||||
const std::shared_ptr<ngraph::Function> ng_function = ngraph::onnx_import::import_onnx_model(resnet50_stream);
|
||||
|
||||
// Check shape of the first output, for example
|
||||
std::cout << ng_function->get_output_shape(0) << std::endl;
|
||||
// The output is Shape{1, 1000}
|
||||
}
|
||||
catch (const ngraph::ngraph_error& error)
|
||||
{
|
||||
std::cout << "Error when importing ONNX model: " << error.what() << std::endl;
|
||||
}
|
||||
}
|
||||
resnet50_stream.close();
|
||||
```
|
||||
@snippet openvino/docs/snippets/OnnxImporterTutorial2.cpp part2
|
||||
|
||||
### <a name="path">Filepath as Input</a>
|
||||
|
||||
The code below shows how to convert the ONNX ResNet50 model to the nGraph function using `import_onnx_model` with the filepath as an input:
|
||||
```cpp
|
||||
const std::shared_ptr<ngraph::Function> ng_function = ngraph::onnx_import::import_onnx_model(resnet50_path);
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/OnnxImporterTutorial3.cpp part3
|
||||
|
||||
[onnx_header]: https://github.com/NervanaSystems/ngraph/blob/master/src/ngraph/frontend/onnx_import/onnx.hpp
|
||||
[onnx_model_zoo]: https://github.com/onnx/models
|
||||
|
||||
@@ -93,40 +93,9 @@ The algorithm for resizing network is the following:
|
||||
3) **Call reshape**
|
||||
|
||||
Here is a code example:
|
||||
```cpp
|
||||
InferenceEngine::Core core;
|
||||
// ------------- 0. Read IR and image ----------------------------------------------
|
||||
CNNNetwork network = core.ReadNetwork("path/to/IR/xml");
|
||||
cv::Mat image = cv::imread("path/to/image");
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
// ------------- 1. Collect the map of input names and shapes from IR---------------
|
||||
auto input_shapes = network.getInputShapes();
|
||||
// ---------------------------------------------------------------------------------
|
||||
@snippet openvino/docs/snippets/ShapeInference.cpp part0
|
||||
|
||||
// ------------- 2. Set new input shapes -------------------------------------------
|
||||
std::string input_name;
|
||||
SizeVector input_shape;
|
||||
std::tie(input_name, input_shape) = *input_shapes.begin(); // let's consider first input only
|
||||
input_shape[0] = batch_size; // set batch size to the first input dimension
|
||||
input_shape[2] = image.rows; // changes input height to the image one
|
||||
input_shape[3] = image.cols; // changes input width to the image one
|
||||
input_shapes[input_name] = input_shape;
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
// ------------- 3. Call reshape ---------------------------------------------------
|
||||
network.reshape(input_shapes);
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
...
|
||||
|
||||
// ------------- 4. Loading model to the device ------------------------------------
|
||||
std::string device = "CPU";
|
||||
ExecutableNetwork executable_network = core.LoadNetwork(network, device);
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
|
||||
```
|
||||
Shape Inference feature is used in [Smart classroom sample](@ref omz_demos_smart_classroom_demo_README).
|
||||
|
||||
## Extensibility
|
||||
|
||||
@@ -20,40 +20,11 @@ following code prepares a graph for shape-relevant parameters.
|
||||
|
||||
> **NOTE**: `validate_nodes_and_infer_types(ops)` must be included for partial shape inference.
|
||||
|
||||
```cpp
|
||||
#include "ngraph/opsets/opset.hpp"
|
||||
#include "ngraph/opsets/opset3.hpp"
|
||||
|
||||
using namespace std;
|
||||
using namespace ngraph;
|
||||
|
||||
auto arg0 = make_shared<opset3::Parameter>(element::f32, Shape{7});
|
||||
auto arg1 = make_shared<opset3::Parameter>(element::f32, Shape{7});
|
||||
// Create an 'Add' operation with two inputs 'arg0' and 'arg1'
|
||||
auto add0 = make_shared<opset3::Add>(arg0, arg1);
|
||||
auto abs0 = make_shared<opset3::Abs>(add0);
|
||||
// Create a node whose inputs/attributes will be specified later
|
||||
auto acos0 = make_shared<opset3::Acos>();
|
||||
// Create a node using opset factories
|
||||
auto add1 = shared_ptr<Node>(get_opset3().create("Add"));
|
||||
// Set inputs to nodes explicitly
|
||||
acos0->set_argument(0, add0);
|
||||
add1->set_argument(0, acos0);
|
||||
add1->set_argument(1, abs0);
|
||||
|
||||
// Run shape inference on the nodes
|
||||
NodeVector ops{arg0, arg1, add0, abs0, acos0, add1};
|
||||
validate_nodes_and_infer_types(ops);
|
||||
|
||||
// Create a graph with one output (add1) and four inputs (arg0, arg1)
|
||||
auto ng_function = make_shared<Function>(OutputVector{add1}, ParameterVector{arg0, arg1});
|
||||
|
||||
```
|
||||
@snippet openvino/docs/snippets/nGraphTutorial.cpp part0
|
||||
|
||||
To wrap it into a CNNNetwork, use:
|
||||
```cpp
|
||||
CNNNetwork net (ng_function);
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/nGraphTutorial.cpp part1
|
||||
|
||||
## Deprecation Notice
|
||||
|
||||
|
||||
@@ -33,14 +33,7 @@ a temporary memory block for model decryption, and use
|
||||
For more information, see the `InferenceEngine::Core` Class
|
||||
Reference Documentation.
|
||||
|
||||
```cpp
|
||||
std::vector<uint8_t> model;
|
||||
std::vector<uint8_t> weights;
|
||||
|
||||
// Read model files and decrypt them into temporary memory block
|
||||
decrypt_file(model_file, password, model);
|
||||
decrypt_file(weights_file, password, weights);
|
||||
```
|
||||
@snippet openvino/docs/snippets/protecting_model_guide.cpp part0
|
||||
|
||||
Hardware-based protection, such as Intel® Software Guard Extensions
|
||||
(Intel® SGX), can be utilized to protect decryption operation secrets and
|
||||
@@ -50,12 +43,7 @@ Extensions](https://software.intel.com/en-us/sgx).
|
||||
Use `InferenceEngine::Core::ReadNetwork()` to set model representations and
|
||||
weights respectively.
|
||||
|
||||
```cpp
|
||||
Core core;
|
||||
// Load model from temporary memory block
|
||||
std::string strModel(model.begin(), model.end());
|
||||
CNNNetwork network = core.ReadNetwork(strModel, make_shared_blob<uint8_t>({Precision::U8, {weights.size()}, C}, weights.data()));
|
||||
```
|
||||
@snippet openvino/docs/snippets/protecting_model_guide.cpp part1
|
||||
|
||||
[deploy_encrypted_model]: img/deploy_encrypted_model.png
|
||||
|
||||
|
||||
@@ -102,124 +102,15 @@ Refer to the sections below to see pseudo-code of usage examples.
|
||||
|
||||
This example uses the OpenCL context obtained from an executable network object.
|
||||
|
||||
```cpp
|
||||
#define CL_HPP_MINIMUM_OPENCL_VERSION 120
|
||||
#define CL_HPP_TARGET_OPENCL_VERSION 120
|
||||
|
||||
#include <CL/cl2.hpp>
|
||||
#include <gpu/gpu_context_api_ocl.hpp>
|
||||
|
||||
...
|
||||
|
||||
// initialize the plugin and load the network
|
||||
InferenceEngine::Core ie;
|
||||
auto exec_net = ie.LoadNetwork(net, "GPU", config);
|
||||
|
||||
// obtain the RemoteContext pointer from the executable network object
|
||||
auto cldnn_context = exec_net.GetContext();
|
||||
// obtain the OpenCL context handle from the RemoteContext,
|
||||
// get device info and create a queue
|
||||
cl::Context ctx = std::dynamic_pointer_cast<ClContext>(cldnn_context);
|
||||
_device = cl::Device(_context.getInfo<CL_CONTEXT_DEVICES>()[0].get(), true);
|
||||
cl::CommandQueue _queue;
|
||||
cl_command_queue_properties props = CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE;
|
||||
_queue = cl::CommandQueue(_context, _device, props);
|
||||
|
||||
// create the OpenCL buffer within the obtained context
|
||||
cl::Buffer shared_buffer(ctx, CL_MEM_READ_WRITE, image_size * num_channels, NULL, &err);
|
||||
// wrap the buffer into RemoteBlob
|
||||
auto shared_blob = gpu::make_shared_blob(input_info->getTensorDesc(), cldnn_context, shared_buffer);
|
||||
|
||||
...
|
||||
// execute user kernel
|
||||
cl::Kernel kernel(program, kernelName.c_str());
|
||||
kernel.setArg(0, shared_buffer);
|
||||
queue.enqueueNDRangeKernel(kernel,
|
||||
cl::NDRange(0),
|
||||
cl::NDRange(image_size),
|
||||
cl::NDRange(1),
|
||||
0, // wait events *
|
||||
&profileEvent);
|
||||
queue.finish();
|
||||
...
|
||||
|
||||
// pass results to the inference
|
||||
inf_req_shared.SetBlob(input_name, shared_blob);
|
||||
inf_req_shared.Infer();
|
||||
|
||||
```
|
||||
@snippet openvino/docs/snippets/GPU_RemoteBlob_API0.cpp part0
|
||||
|
||||
### Running GPU Plugin Inference within User-Supplied Shared Context
|
||||
|
||||
```cpp
|
||||
#define CL_HPP_MINIMUM_OPENCL_VERSION 120
|
||||
#define CL_HPP_TARGET_OPENCL_VERSION 120
|
||||
@snippet openvino/docs/snippets/GPU_RemoteBlob_API1.cpp part1
|
||||
|
||||
#include <CL/cl2.hpp>
|
||||
#include <gpu/gpu_context_api_ocl.hpp>
|
||||
|
||||
...
|
||||
|
||||
cl::Context ctx = get_my_OpenCL_context();
|
||||
|
||||
// share the context with GPU plugin and compile ExecutableNetwork
|
||||
auto remote_context = gpu::make_shared_context(ie, "GPU", ocl_instance->_context.get());
|
||||
auto exec_net_shared = ie.LoadNetwork(net, remote_context);
|
||||
auto inf_req_shared = exec_net_shared.CreateInferRequest();
|
||||
|
||||
...
|
||||
// do OpenCL processing stuff
|
||||
...
|
||||
|
||||
// run the inference
|
||||
inf_req_shared.Infer();
|
||||
|
||||
```
|
||||
### Direct Consuming of the NV12 VAAPI Video Decoder Surface on Linux
|
||||
|
||||
```cpp
|
||||
#include <gpu/gpu_context_api_va.hpp>
|
||||
#include <cldnn/cldnn_config.hpp>
|
||||
|
||||
...
|
||||
|
||||
// initialize the objects
|
||||
CNNNetwork network = ie.ReadNetwork(xmlFileName, binFileName);
|
||||
|
||||
...
|
||||
|
||||
auto inputInfoItem = *inputInfo.begin();
|
||||
inputInfoItem.second->setPrecision(Precision::U8);
|
||||
inputInfoItem.second->setLayout(Layout::NCHW);
|
||||
inputInfoItem.second->getPreProcess().setColorFormat(ColorFormat::NV12);
|
||||
|
||||
VADisplay disp = get_VA_Device();
|
||||
// create the shared context object
|
||||
auto shared_va_context = gpu::make_shared_context(ie, "GPU", disp);
|
||||
// compile network within a shared context
|
||||
ExecutableNetwork executable_network = ie.LoadNetwork(network,
|
||||
shared_va_context,
|
||||
{ { CLDNNConfigParams::KEY_CLDNN_NV12_TWO_INPUTS,
|
||||
PluginConfigParams::YES } });
|
||||
|
||||
// decode/inference loop
|
||||
for (int i = 0; i < nframes; i++) {
|
||||
...
|
||||
// execute decoding and obtain decoded surface handle
|
||||
decoder.DecodeFrame();
|
||||
VASurfaceID va_surface = decoder.get_VA_output_surface();
|
||||
...
|
||||
//wrap decoder output into RemoteBlobs and set it as inference input
|
||||
auto nv12_blob = gpu::make_shared_blob_nv12(ieInHeight,
|
||||
ieInWidth,
|
||||
shared_va_context,
|
||||
va_surface
|
||||
);
|
||||
inferRequests[currentFrame].SetBlob(input_name, nv12_blob);
|
||||
inferRequests[currentFrame].StartAsync();
|
||||
inferRequests[prevFrame].Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY);
|
||||
}
|
||||
```
|
||||
@snippet openvino/docs/snippets/GPU_RemoteBlob_API2.cpp part2
|
||||
|
||||
## See Also
|
||||
|
||||
|
||||
@@ -28,43 +28,15 @@ Default fallback policy decides which layer goes to which device automatically a
|
||||
|
||||
Another way to annotate a network is to set affinity manually using <code>ngraph::Node::get_rt_info</code> with key `"affinity"`:
|
||||
|
||||
```cpp
|
||||
for (auto && op : function->get_ops())
|
||||
op->get_rt_info()["affinity"] = std::shared_ptr<ngraph::VariantWrapper<std::string>>("CPU");
|
||||
```
|
||||
@snippet openvino/docs/snippets/HETERO0.cpp part0
|
||||
|
||||
The fallback policy does not work if even one layer has an initialized affinity. The sequence should be calling of automating affinity settings and then fix manually.
|
||||
```cpp
|
||||
InferenceEngine::Core core
|
||||
auto network = core.ReadNetwork("Model.xml");
|
||||
|
||||
// This example demonstrates how to perform default affinity initialization and then
|
||||
// correct affinity manually for some layers
|
||||
const std::string device = "HETERO:FPGA,CPU";
|
||||
|
||||
// QueryNetworkResult object contains map layer -> device
|
||||
InferenceEngine::QueryNetworkResult res = core.QueryNetwork(network, device, { });
|
||||
|
||||
// update default affinities
|
||||
res.supportedLayersMap["layerName"] = "CPU";
|
||||
|
||||
// set affinities to network
|
||||
for (auto&& node : function->get_ops()) {
|
||||
auto& affinity = res.supportedLayersMap[node->get_friendly_name()];
|
||||
// Store affinity mapping using node runtime information
|
||||
node->get_rt_info()["affinity"] = std::make_shared<ngraph::VariantWrapper<std::string>>(affinity);
|
||||
}
|
||||
|
||||
// load network with affinities set before
|
||||
auto executable_network = core.LoadNetwork(network, device);
|
||||
```
|
||||
@snippet openvino/docs/snippets/HETERO1.cpp part1
|
||||
|
||||
If you rely on the default affinity distribution, you can avoid calling <code>InferenceEngine::Core::QueryNetwork</code> and just call <code>InferenceEngine::Core::LoadNetwork</code> instead:
|
||||
```cpp
|
||||
InferenceEngine::Core core
|
||||
auto network = core.ReadNetwork("Model.xml");
|
||||
auto executable_network = core.LoadNetwork(network, "HETERO:FPGA,CPU");
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/HETERO2.cpp part2
|
||||
|
||||
> **NOTE**: `InferenceEngine::Core::QueryNetwork` does not depend on affinities set by a user, but queries for layer support based on device capabilities.
|
||||
|
||||
@@ -100,16 +72,7 @@ Heterogeneous plugin can generate two files:
|
||||
* `hetero_affinity_<network name>.dot` - annotation of affinities per layer. This file is written to the disk only if default fallback policy was executed
|
||||
* `hetero_subgraphs_<network name>.dot` - annotation of affinities per graph. This file is written to the disk during execution of <code>ICNNNetwork::LoadNetwork()</code> for heterogeneous plugin
|
||||
|
||||
```cpp
|
||||
#include "ie_plugin_config.hpp"
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
using namespace InferenceEngine::PluginConfigParams;
|
||||
using namespace InferenceEngine::HeteroConfigParams;
|
||||
|
||||
...
|
||||
InferenceEngine::Core core;
|
||||
core.SetConfig({ { KEY_HETERO_DUMP_GRAPH_DOT, YES } }, "HETERO");
|
||||
```
|
||||
@snippet openvino/docs/snippets/HETERO3.cpp part3
|
||||
|
||||
You can use GraphViz* utility or converters to `.png` formats. On Ubuntu* operating system, you can use the following utilities:
|
||||
* `sudo apt-get install xdot`
|
||||
|
||||
@@ -31,33 +31,13 @@ The only configuration option for the multi-device is prioritized list of device
|
||||
You can use name of the configuration directly as a string, or use MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES from the multi/multi_device_config.hpp that defines the same string.
|
||||
|
||||
Basically, there are three ways to specify the devices to be use by the "MULTI":
|
||||
```cpp
|
||||
Core ie;
|
||||
//NEW IE-CENTRIC API, the "MULTI" plugin is (globally) pre-configured with the explicit option:
|
||||
ie.SetConfig({{"MULTI_DEVICE_PRIORITIES", "HDDL,GPU"}}, "MULTI");
|
||||
ExecutableNetwork exec0 = ie.LoadNetwork(network, "MULTI", {});
|
||||
|
||||
//NEW IE-CENTRIC API, configuration of the "MULTI" is part of the network configuration (and hence specific to the network):
|
||||
ExecutableNetwork exec1 = ie.LoadNetwork(network, "MULTI", {{"MULTI_DEVICE_PRIORITIES", "HDDL,GPU"}});
|
||||
//NEW IE-CENTRIC API, same as previous, but configuration of the "MULTI" is part of the name (so config is empty), also network-specific:
|
||||
ExecutableNetwork exec2 = ie.LoadNetwork(network, "MULTI:HDDL,GPU", {});
|
||||
```
|
||||
@snippet openvino/docs/snippets/MULTI0.cpp part0
|
||||
|
||||
Notice that the priorities of the devices can be changed in real-time for the executable network:
|
||||
```cpp
|
||||
Core ie;
|
||||
ExecutableNetwork exec = ie.LoadNetwork(network, "MULTI:HDDL,GPU", {});
|
||||
//...
|
||||
exec.SetConfig({{"MULTI_DEVICE_PRIORITIES", "GPU,HDDL"}});
|
||||
// you can even exclude some device
|
||||
exec.SetConfig({{"MULTI_DEVICE_PRIORITIES", "GPU"}});
|
||||
//...
|
||||
// and then return it back
|
||||
exec.SetConfig({{"MULTI_DEVICE_PRIORITIES", "GPU,HDDL"}});
|
||||
//but you cannot add new devices on the fly, the next line will trigger the following exception:
|
||||
//[ ERROR ] [NOT_FOUND] You can only change device priorities but not add new devices with the Network's SetConfig(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES.
|
||||
//CPU device was not in the original device list!
|
||||
exec.SetConfig({{"MULTI_DEVICE_PRIORITIES", "CPU,GPU,HDDL"}});
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/MULTI1.cpp part1
|
||||
|
||||
Finally, there is a way to specify number of requests that the multi-device will internally keep for each device.
|
||||
Say if your original app was running 4 cameras with 4 inference requests now you would probably want to share these 4 requests between 2 devices used in the MULTI. The easiest way is to specify a number of requests for each device using parentheses: "MULTI:CPU(2),GPU(2)" and use the same 4 requests in your app. However, such an explicit configuration is not performance portable and hence not recommended. Instead, the better way is to configure the individual devices and query the resulting number of requests to be used in the application level (see [Configuring the Individual Devices and Creating the Multi-Device On Top](#configuring-the-individual-devices-and-creating-the-multi-device-on-top)).
|
||||
|
||||
@@ -74,16 +54,9 @@ Available devices:
|
||||
Device: HDDL
|
||||
```
|
||||
Simple programmatic way to enumerate the devices and use with the multi-device is as follows:
|
||||
```cpp
|
||||
Core ie;
|
||||
std::string allDevices = "MULTI:";
|
||||
std::vector<std::string> availableDevices = ie.GetAvailableDevices();
|
||||
for (auto && device : availableDevices) {
|
||||
allDevices += device;
|
||||
allDevices += ((device == availableDevices[availableDevices.size()-1]) ? "" : ",");
|
||||
}
|
||||
ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, allDevices, {});
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/MULTI2.cpp part2
|
||||
|
||||
Beyond trivial "CPU", "GPU", "HDDL" and so on, when multiple instances of a device are available the names are more qualified.
|
||||
For example this is how two Intel® Movidius™ Myriad™ X sticks are listed with the hello_query_sample:
|
||||
```
|
||||
@@ -94,33 +67,15 @@ For example this is how two Intel® Movidius™ Myriad™ X sticks are listed wi
|
||||
```
|
||||
So the explicit configuration to use both would be "MULTI:MYRIAD.1.2-ma2480,MYRIAD.1.4-ma2480".
|
||||
Accordingly, the code that loops over all available devices of "MYRIAD" type only is below:
|
||||
```cpp
|
||||
Core ie;
|
||||
std::string allDevices = "MULTI:";
|
||||
std::vector<std::string> myriadDevices = ie->GetMetric("MYRIAD", METRIC_KEY(myriadDevices)));
|
||||
for (int i = 0; i < myriadDevices.size(); ++i) {
|
||||
allDevices += std::string("MYRIAD.")
|
||||
+ myriadDevices[i]
|
||||
+ std::string(i < (myriadDevices.size() -1) ? "," : "");
|
||||
}
|
||||
ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, allDevices, {});
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/MULTI3.cpp part3
|
||||
|
||||
|
||||
## Configuring the Individual Devices and Creating the Multi-Device On Top
|
||||
As discussed in the first section, you shall configure each individual device as usual and then just create the "MULTI" device on top:
|
||||
```cpp
|
||||
#include <multi/multi_device_config.hpp>
|
||||
// configure the HDDL device first
|
||||
Core ie;
|
||||
ie.SetConfig(hddl_config, "HDDL");
|
||||
// configure the GPU device
|
||||
ie.SetConfig(gpu_config, "GPU");
|
||||
// load the network to the multi-device, while specifying the configuration (devices along with priorities):
|
||||
ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, "MULTI", {{MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, "HDDL,GPU"}});
|
||||
// new metric allows to query the optimal number of requests:
|
||||
uint32_t nireq = exeNetwork.GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<unsigned int>();
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/MULTI4.cpp part4
|
||||
|
||||
Alternatively, you can combine all the individual device settings into single config and load that, allowing the multi-device plugin to parse and apply that to the right devices. See code example in the next section.
|
||||
|
||||
Notice that while the performance of accelerators combines really well with multi-device, the CPU+GPU execution poses some performance caveats, as these devices share the power, bandwidth and other resources. For example it is recommended to enable the GPU throttling hint (which save another CPU thread for the CPU inference).
|
||||
@@ -128,12 +83,8 @@ See section of the [Using the multi-device with OpenVINO samples and benchmarkin
|
||||
|
||||
## Querying the Optimal Number of Inference Requests
|
||||
Notice that until R2 you had to calculate number of requests in your application for any device, e.g. you had to know that Intel® Vision Accelerator Design with Intel® Movidius™ VPUs required at least 32 inference requests to perform well. Now you can use the new GetMetric API to query the optimal number of requests. Similarly, when using the multi-device you don't need to sum over included devices yourself, you can query metric directly:
|
||||
```cpp
|
||||
// 'device_name' can be "MULTI:HDDL,GPU" to configure the multi-device to use HDDL and GPU
|
||||
ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, device_name, full_config);
|
||||
// new metric allows to query the optimal number of requests:
|
||||
uint32_t nireq = exeNetwork.GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<unsigned int>();
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/MULTI5.cpp part5
|
||||
|
||||
## Using the Multi-Device with OpenVINO Samples and Benchmarking the Performance
|
||||
Notice that every OpenVINO sample that supports "-d" (which stays for "device") command-line option transparently accepts the multi-device.
|
||||
|
||||
@@ -17,19 +17,12 @@ The following section provides information on how to distribute a model across a
|
||||
The structure should hold:
|
||||
1. A pointer to an inference request.
|
||||
2. An ID to keep track of the request.
|
||||
```cpp
|
||||
struct Request {
|
||||
InferenceEngine::InferRequest::Ptr inferRequest;
|
||||
int frameidx;
|
||||
};
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/movidius-programming-guide.cpp part0
|
||||
|
||||
### Declare a Vector of Requests
|
||||
|
||||
```cpp
|
||||
// numRequests is the number of frames (max size, equal to the number of VPUs in use)
|
||||
vector<Request> request(numRequests);
|
||||
```
|
||||
@snippet openvino/docs/snippets/movidius-programming-guide.cpp part1
|
||||
|
||||
Declare and initialize 2 mutex variables:
|
||||
1. For each request
|
||||
@@ -41,15 +34,9 @@ Conditional variable indicates when at most 8 requests are done at a time.
|
||||
|
||||
For inference requests, use the asynchronous IE API calls:
|
||||
|
||||
```cpp
|
||||
// initialize infer request pointer – Consult IE API for more detail.
|
||||
request[i].inferRequest = executable_network.CreateInferRequestPtr();
|
||||
```
|
||||
@snippet openvino/docs/snippets/movidius-programming-guide.cpp part2
|
||||
|
||||
```cpp
|
||||
// Run inference
|
||||
request[i].inferRequest->StartAsync();
|
||||
```
|
||||
@snippet openvino/docs/snippets/movidius-programming-guide.cpp part3
|
||||
|
||||
|
||||
### Create a Lambda Function
|
||||
@@ -58,10 +45,7 @@ Lambda Function enables the parsing and display of results.
|
||||
|
||||
Inside the Lambda body use the completion callback function:
|
||||
|
||||
```cpp
|
||||
request[i].inferRequest->SetCompletionCallback
|
||||
(nferenceEngine::IInferRequest::Ptr context)
|
||||
```
|
||||
@snippet openvino/docs/snippets/movidius-programming-guide.cpp part4
|
||||
|
||||
## Additional Resources
|
||||
|
||||
|
||||
@@ -265,15 +265,7 @@ The following tips are provided to give general guidance on optimizing execution
|
||||
|
||||
There is a dedicated configuration option that enables dumping the visualization of the subgraphs created by the heterogeneous plugin:
|
||||
|
||||
```cpp
|
||||
#include "ie_plugin_config.hpp"
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
using namespace InferenceEngine::PluginConfigParams;
|
||||
using namespace InferenceEngine::HeteroConfigParams;
|
||||
|
||||
...
|
||||
auto execNetwork = ie.LoadNetwork(network, "HETERO:FPGA,CPU", { {KEY_HETERO_DUMP_GRAPH_DOT, YES} });
|
||||
```
|
||||
@snippet openvino/docs/snippets/dldt_optimization_guide0.cpp part0
|
||||
|
||||
After enabling the configuration key, the heterogeneous plugin generates two files:
|
||||
|
||||
@@ -341,11 +333,8 @@ If you are building an app-level pipeline with third-party components like GStre
|
||||
In many cases, a network expects a pre-processed image, so make sure you do not perform unnecessary steps in your code:
|
||||
- Model Optimizer can efficiently bake the mean and normalization (scale) values into the model (for example, weights of the first convolution). See <a href="#mo-knobs-related-to-performance">Model Optimizer Knobs Related to Performance</a>.
|
||||
- If regular 8-bit per channel images are your native media (for instance, decoded frames), do not convert to the `FP32` on your side, as this is something that plugins can accelerate. Use the `InferenceEngine::Precision::U8` as your input format:<br>
|
||||
```cpp
|
||||
InferenceEngine::InputsDataMap info(netReader.getNetwork().getInputsInfo());
|
||||
auto& inputInfoFirst = info.begin()->second;
|
||||
info->setInputPrecision(Precision::U8);
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/dldt_optimization_guide1.cpp part1
|
||||
|
||||
Note that in many cases, you can directly share the (input) data with the Inference Engine.
|
||||
|
||||
@@ -354,47 +343,16 @@ Note that in many cases, you can directly share the (input) data with the Infere
|
||||
The general approach for sharing data between Inference Engine and media/graphics APIs like Intel® Media Server Studio (Intel® MSS) is based on sharing the *system* memory. That is, in your code, you should map or copy the data from the API to the CPU address space first.
|
||||
|
||||
For Intel MSS, it is recommended to perform a viable pre-processing, for example, crop/resize, and then convert to RGB again with the [Video Processing Procedures (VPP)](https://software.intel.com/en-us/node/696108). Then lock the result and create an Inference Engine blob on top of that. The resulting pointer can be used for the `SetBlob`:
|
||||
```cpp
|
||||
//Lock Intel MSS surface
|
||||
mfxFrameSurface1 *frame_in; //Input MSS surface.
|
||||
mfxFrameAllocator* pAlloc = &m_mfxCore.FrameAllocator();
|
||||
pAlloc->Lock(pAlloc->pthis, frame_in->Data.MemId, &frame_in->Data);
|
||||
//Inference Engine code
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/dldt_optimization_guide2.cpp part2
|
||||
|
||||
**WARNING**: The `InferenceEngine::NHWC` layout is not supported natively by most InferenceEngine plugins so internal conversion might happen.
|
||||
|
||||
```cpp
|
||||
InferenceEngine::SizeVector dims_src = {
|
||||
1 /* batch, N*/,
|
||||
(size_t) frame_in->Info.Height /* Height */,
|
||||
(size_t) frame_in->Info.Width /* Width */,
|
||||
3 /*Channels,*/,
|
||||
};
|
||||
TensorDesc desc(InferenceEngine::Precision::U8, dims_src, InferenceEngine::NHWC);
|
||||
/* wrapping the surface data, as RGB is interleaved, need to pass only ptr to the R, notice that this wouldn’t work with planar formats as these are 3 separate planes/pointers*/
|
||||
InferenceEngine::TBlob<uint8_t>::Ptr p = InferenceEngine::make_shared_blob<uint8_t>( desc, (uint8_t*) frame_in->Data.R);
|
||||
inferRequest.SetBlob(“input”, p);
|
||||
inferRequest.Infer();
|
||||
//Make sure to unlock the surface upon inference completion, to return the ownership back to the Intel MSS
|
||||
pAlloc->Unlock(pAlloc->pthis, frame_in->Data.MemId, &frame_in->Data);
|
||||
```
|
||||
@snippet openvino/docs/snippets/dldt_optimization_guide3.cpp part3
|
||||
|
||||
Alternatively, you can use RGBP (planar RGB) output from Intel MSS. This allows to wrap the (locked) result as regular NCHW which is generally friendly for most plugins (unlike NHWC). Then you can use it with `SetBlob` just like in previous example:
|
||||
|
||||
```cpp
|
||||
InferenceEngine::SizeVector dims_src = {
|
||||
1 /* batch, N*/,
|
||||
3 /*Channels,*/,
|
||||
(size_t) frame_in->Info.Height /* Height */,
|
||||
(size_t) frame_in->Info.Width /* Width */,
|
||||
};
|
||||
TensorDesc desc(InferenceEngine::Precision::U8, dims_src, InferenceEngine::NCHW);
|
||||
/* wrapping the RGBP surface data*/
|
||||
InferenceEngine::TBlob<uint8_t>::Ptr p = InferenceEngine::make_shared_blob<uint8_t>( desc, (uint8_t*) frame_in->Data.R);
|
||||
inferRequest.SetBlob("input", p);
|
||||
…
|
||||
```
|
||||
@snippet openvino/docs/snippets/dldt_optimization_guide4.cpp part4
|
||||
|
||||
The only downside of this approach is that VPP conversion to RGBP is not hardware accelerated (and performed on the GPU EUs). Also, it is available only on LInux.
|
||||
|
||||
@@ -406,27 +364,7 @@ Again, if the OpenCV and Inference Engine layouts match, the data can be wrapped
|
||||
|
||||
**WARNING**: The `InferenceEngine::NHWC` layout is not supported natively by most InferenceEngine plugins so internal conversion might happen.
|
||||
|
||||
```cpp
|
||||
cv::Mat frame = ...; // regular CV_8UC3 image, interleaved
|
||||
// creating blob that wraps the OpenCV’s Mat
|
||||
// (the data it points should persists until the blob is released):
|
||||
InferenceEngine::SizeVector dims_src = {
|
||||
1 /* batch, N*/,
|
||||
(size_t)frame.rows /* Height */,
|
||||
(size_t)frame.cols /* Width */,
|
||||
(size_t)frame.channels() /*Channels,*/,
|
||||
};
|
||||
TensorDesc desc(InferenceEngine::Precision::U8, dims_src, InferenceEngine::NHWC);
|
||||
InferenceEngine::TBlob<uint8_t>::Ptr p = InferenceEngine::make_shared_blob<uint8_t>( desc, (uint8_t*)frame.data, frame.step[0] * frame.rows);
|
||||
inferRequest.SetBlob(“input”, p);
|
||||
inferRequest.Infer();
|
||||
…
|
||||
// similarly, you can wrap the output tensor (let’s assume it is FP32)
|
||||
// notice that the output should be also explicitly stated as NHWC with setLayout
|
||||
const float* output_data = output_blob->buffer().
|
||||
as<PrecisionTrait<Precision::FP32>::value_type*>();
|
||||
cv::Mat res (rows, cols, CV_32FC3, output_data, CV_AUTOSTEP);
|
||||
```
|
||||
@snippet openvino/docs/snippets/dldt_optimization_guide5.cpp part5
|
||||
|
||||
Notice that original `cv::Mat`/blobs cannot be used simultaneously by the application and the Inference Engine. Alternatively, the data that the pointer references to can be copied to unlock the original data and return ownership to the original API.
|
||||
|
||||
@@ -436,25 +374,7 @@ Infer Request based API offers two types of request: Sync and Async. The Sync is
|
||||
|
||||
More importantly, an infer request encapsulates the reference to the “executable” network and actual inputs/outputs. Now, when you load the network to the plugin, you get a reference to the executable network (you may consider that as a queue). Actual infer requests are created by the executable network:
|
||||
|
||||
```cpp
|
||||
Core ie;
|
||||
auto network = ie.ReadNetwork("Model.xml", "Model.bin");
|
||||
InferenceEngine::InputsDataMap input_info(network.getInputsInfo());
|
||||
|
||||
auto executable_network = ie.LoadNetwork(network, "GPU");
|
||||
auto infer_request = executable_network.CreateInferRequest();
|
||||
|
||||
for (auto & item : inputInfo) {
|
||||
std::string input_name = item->first;
|
||||
auto input = infer_request.GetBlob(input_name);
|
||||
/** Lock/Fill input tensor with data **/
|
||||
unsigned char* data =
|
||||
input->buffer().as<PrecisionTrait<Precision::U8>::value_type*>();
|
||||
...
|
||||
}
|
||||
|
||||
infer_request->Infer();
|
||||
```
|
||||
@snippet openvino/docs/snippets/dldt_optimization_guide6.cpp part6
|
||||
|
||||
`GetBlob` is a recommend way to communicate with the network, as it internally allocates the data with right padding/alignment for the device. For example, the GPU inputs/outputs blobs are mapped to the host (which is fast) if the `GetBlob` is used. But if you called the `SetBlob`, the copy (from/to the blob you have set) into the internal GPU plugin structures will happen.
|
||||
|
||||
@@ -464,11 +384,9 @@ If your application simultaneously executes multiple infer requests:
|
||||
|
||||
- For the CPU, the best solution, you can use the <a href="#cpu-streams">CPU "throughput" mode</a>.
|
||||
- If latency is of more concern, you can try the `EXCLUSIVE_ASYNC_REQUESTS` [configuration option](../IE_DG/supported_plugins/CPU.md) that limits the number of the simultaneously executed requests for all (executable) networks that share the specific device to just one:<br>
|
||||
```cpp
|
||||
//these two networks go thru same plugin (aka device) and their requests will not overlap.
|
||||
auto executable_network0 = plugin.LoadNetwork(network0, {{PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, PluginConfigParams::YES}});
|
||||
auto executable_network1 = plugin.LoadNetwork(network1, {{PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, PluginConfigParams::YES}});
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/dldt_optimization_guide7.cpp part7
|
||||
|
||||
<br>For more information on the executable networks notation, see <a href="#new-request-based-api">Request-Based API and “GetBlob” Idiom</a>.
|
||||
|
||||
- The heterogeneous device uses the `EXCLUSIVE_ASYNC_REQUESTS` by default.
|
||||
@@ -490,27 +408,15 @@ In the example below, inference is applied to the results of the video decoding.
|
||||
You can compare the pseudo-codes for the regular and async-based approaches:
|
||||
|
||||
- In the regular way, the frame is captured with OpenCV and then immediately processed:<br>
|
||||
```cpp
|
||||
while(…) {
|
||||
capture frame
|
||||
populate CURRENT InferRequest
|
||||
Infer CURRENT InferRequest //this call is synchronous
|
||||
display CURRENT result
|
||||
}
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/dldt_optimization_guide8.cpp part8
|
||||
|
||||
|
||||
|
||||
- In the "true" async mode, the `NEXT` request is populated in the main (application) thread, while the `CURRENT` request is processed:<br>
|
||||
```cpp
|
||||
while(…) {
|
||||
capture frame
|
||||
populate NEXT InferRequest
|
||||
start NEXT InferRequest //this call is async and returns immediately
|
||||
wait for the CURRENT InferRequest //processed in a dedicated thread
|
||||
display CURRENT result
|
||||
swap CURRENT and NEXT InferRequests
|
||||
}
|
||||
```
|
||||
|
||||
@snippet openvino/docs/snippets/dldt_optimization_guide9.cpp part9
|
||||
|
||||
|
||||
|
||||
The technique can be generalized to any available parallel slack. For example, you can do inference and simultaneously encode the resulting or previous frames or run further inference, like emotion detection on top of the face detection results.
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
#include <inference_engine.hpp>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part0]
|
||||
InferenceEngine::Core core;
|
||||
|
||||
auto cpuOptimizationCapabilities = core.GetMetric("CPU", METRIC_KEY(OPTIMIZATION_CAPABILITIES)).as<std::vector<std::string>>();
|
||||
//! [part0]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
#include <inference_engine.hpp>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part1]
|
||||
InferenceEngine::Core core;
|
||||
|
||||
auto network = core.ReadNetwork("sample.xml");
|
||||
|
||||
auto exeNetwork = core.LoadNetwork(network, "CPU");
|
||||
|
||||
auto enforceBF16 = exeNetwork.GetConfig(PluginConfigParams::KEY_ENFORCE_BF16).as<std::string>();
|
||||
//! [part1]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#include <inference_engine.hpp>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part2]
|
||||
InferenceEngine::Core core;
|
||||
|
||||
core.SetConfig({ { CONFIG_KEY(ENFORCE_BF16), CONFIG_VALUE(NO) } }, "CPU");
|
||||
//! [part2]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
#include <inference_engine.hpp>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part0]
|
||||
InferenceEngine::Core core;
|
||||
// Load CPU extension as a shared library
|
||||
auto extension_ptr = make_so_pointer<InferenceEngine::IExtension>("<shared lib path>");
|
||||
// Add extension to the CPU device
|
||||
core.AddExtension(extension_ptr, "CPU");
|
||||
//! [part0]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <vector>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
int FLAGS_bl = 1;
|
||||
auto imagesData = std::vector<std::string>(2);
|
||||
auto imagesData2 = std::vector<std::string>(4);
|
||||
//! [part0]
|
||||
int dynBatchLimit = FLAGS_bl; //take dynamic batch limit from command line option
|
||||
|
||||
|
||||
// Read network model
|
||||
Core core;
|
||||
CNNNetwork network = core.ReadNetwork("sample.xml");
|
||||
|
||||
|
||||
// enable dynamic batching and prepare for setting max batch limit
|
||||
const std::map<std::string, std::string> dyn_config =
|
||||
{ { PluginConfigParams::KEY_DYN_BATCH_ENABLED, PluginConfigParams::YES } };
|
||||
network.setBatchSize(dynBatchLimit);
|
||||
|
||||
|
||||
// create executable network and infer request
|
||||
auto executable_network = core.LoadNetwork(network, "CPU", dyn_config);
|
||||
auto infer_request = executable_network.CreateInferRequest();
|
||||
|
||||
|
||||
// ...
|
||||
|
||||
|
||||
// process a set of images
|
||||
// dynamically set batch size for subsequent Infer() calls of this request
|
||||
size_t batchSize = imagesData.size();
|
||||
infer_request.SetBatch(batchSize);
|
||||
infer_request.Infer();
|
||||
|
||||
|
||||
// ...
|
||||
|
||||
|
||||
// process another set of images
|
||||
batchSize = imagesData2.size();
|
||||
infer_request.SetBatch(batchSize);
|
||||
infer_request.Infer();
|
||||
//! [part0]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
#include <inference_engine.hpp>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part0]
|
||||
InferenceEngine::Core core;
|
||||
// Load GPU Extensions
|
||||
core.SetConfig({ { InferenceEngine::PluginConfigParams::KEY_CONFIG_FILE, "<path_to_the_xml_file>" } }, "GPU");
|
||||
//! [part0]
|
||||
|
||||
//! [part1]
|
||||
core.SetConfig({ { PluginConfigParams::KEY_DUMP_KERNELS, PluginConfigParams::YES } }, "GPU");
|
||||
//! [part1]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
#include <inference_engine.hpp>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part0]
|
||||
Core ie;
|
||||
ie.SetConfig({{ CONFIG_KEY(TUNING_MODE), CONFIG_VALUE(TUNING_CREATE) }}, "GPU");
|
||||
ie.SetConfig({{ CONFIG_KEY(TUNING_FILE), "/path/to/tuning/file.json" }}, "GPU");
|
||||
// Further LoadNetwork calls will use the specified tuning parameters
|
||||
//! [part0]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
#define CL_HPP_MINIMUM_OPENCL_VERSION 120
|
||||
#define CL_HPP_TARGET_OPENCL_VERSION 120
|
||||
|
||||
#include <inference_engine.hpp>
|
||||
#include <CL/cl2.hpp>
|
||||
#include <gpu/gpu_context_api_ocl.hpp>
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part0]
|
||||
|
||||
|
||||
// ...
|
||||
|
||||
|
||||
// initialize the plugin and load the network
|
||||
InferenceEngine::Core ie;
|
||||
auto exec_net = ie.LoadNetwork(net, "GPU", config);
|
||||
|
||||
|
||||
// obtain the RemoteContext pointer from the executable network object
|
||||
auto cldnn_context = exec_net.GetContext();
|
||||
// obtain the OpenCL context handle from the RemoteContext,
|
||||
// get device info and create a queue
|
||||
cl::Context ctx = std::dynamic_pointer_cast<ClContext>(cldnn_context);
|
||||
_device = cl::Device(_context.getInfo<CL_CONTEXT_DEVICES>()[0].get(), true);
|
||||
cl::CommandQueue _queue;
|
||||
cl_command_queue_properties props = CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE;
|
||||
_queue = cl::CommandQueue(_context, _device, props);
|
||||
|
||||
|
||||
// create the OpenCL buffer within the obtained context
|
||||
cl::Buffer shared_buffer(ctx, CL_MEM_READ_WRITE, image_size * num_channels, NULL, &err);
|
||||
// wrap the buffer into RemoteBlob
|
||||
auto shared_blob = gpu::make_shared_blob(input_info->getTensorDesc(), cldnn_context, shared_buffer);
|
||||
|
||||
|
||||
// ...
|
||||
// execute user kernel
|
||||
cl::Kernel kernel(program, kernelName.c_str());
|
||||
kernel.setArg(0, shared_buffer);
|
||||
queue.enqueueNDRangeKernel(kernel,
|
||||
cl::NDRange(0),
|
||||
cl::NDRange(image_size),
|
||||
cl::NDRange(1),
|
||||
0, // wait events *
|
||||
&profileEvent);
|
||||
queue.finish();
|
||||
// ...
|
||||
|
||||
|
||||
// pass results to the inference
|
||||
inf_req_shared.SetBlob(input_name, shared_blob);
|
||||
inf_req_shared.Infer();
|
||||
//! [part0]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
#define CL_HPP_MINIMUM_OPENCL_VERSION 120
|
||||
#define CL_HPP_TARGET_OPENCL_VERSION 120
|
||||
|
||||
#include <inference_engine.hpp>
|
||||
#include <CL/cl2.hpp>
|
||||
#include <gpu/gpu_context_api_ocl.hpp>
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part1]
|
||||
|
||||
// ...
|
||||
|
||||
cl::Context ctx = get_my_OpenCL_context();
|
||||
|
||||
// share the context with GPU plugin and compile ExecutableNetwork
|
||||
auto remote_context = gpu::make_shared_context(ie, "GPU", ocl_instance->_context.get());
|
||||
auto exec_net_shared = ie.LoadNetwork(net, remote_context);
|
||||
auto inf_req_shared = exec_net_shared.CreateInferRequest();
|
||||
|
||||
|
||||
// ...
|
||||
// do OpenCL processing stuff
|
||||
// ...
|
||||
|
||||
// run the inference
|
||||
inf_req_shared.Infer();
|
||||
//! [part1]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <gpu/gpu_context_api_va.hpp>
|
||||
#include <cldnn/cldnn_config.hpp>
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part2]
|
||||
|
||||
// ...
|
||||
|
||||
|
||||
// initialize the objects
|
||||
CNNNetwork network = ie.ReadNetwork(xmlFileName, binFileName);
|
||||
|
||||
|
||||
// ...
|
||||
|
||||
|
||||
auto inputInfoItem = *inputInfo.begin();
|
||||
inputInfoItem.second->setPrecision(Precision::U8);
|
||||
inputInfoItem.second->setLayout(Layout::NCHW);
|
||||
inputInfoItem.second->getPreProcess().setColorFormat(ColorFormat::NV12);
|
||||
|
||||
VADisplay disp = get_VA_Device();
|
||||
// create the shared context object
|
||||
auto shared_va_context = gpu::make_shared_context(ie, "GPU", disp);
|
||||
// compile network within a shared context
|
||||
ExecutableNetwork executable_network = ie.LoadNetwork(network,
|
||||
shared_va_context,
|
||||
{ { CLDNNConfigParams::KEY_CLDNN_NV12_TWO_INPUTS,
|
||||
PluginConfigParams::YES } });
|
||||
|
||||
|
||||
// decode/inference loop
|
||||
for (int i = 0; i < nframes; i++) {
|
||||
// ...
|
||||
// execute decoding and obtain decoded surface handle
|
||||
decoder.DecodeFrame();
|
||||
VASurfaceID va_surface = decoder.get_VA_output_surface();
|
||||
// ...
|
||||
//wrap decoder output into RemoteBlobs and set it as inference input
|
||||
auto nv12_blob = gpu::make_shared_blob_nv12(ieInHeight,
|
||||
ieInWidth,
|
||||
shared_va_context,
|
||||
va_surface
|
||||
);
|
||||
inferRequests[currentFrame].SetBlob(input_name, nv12_blob);
|
||||
inferRequests[currentFrame].StartAsync();
|
||||
inferRequests[prevFrame].Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY);
|
||||
}
|
||||
//! [part2]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <ngraph/pass/visualize_tree.hpp>
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part0]
|
||||
std::shared_ptr<ngraph::Function> nGraph;
|
||||
// ...
|
||||
ngraph::pass::VisualizeTree("after.png").run_on_function(nGraph); // Visualize the nGraph function to an image
|
||||
//! [part0]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <ngraph/pass/visualize_tree.hpp>
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part1]
|
||||
std::shared_ptr<ngraph::Function> nGraph;
|
||||
// ...
|
||||
CNNNetwork network(nGraph);
|
||||
network.serialize("test_ir.xml", "test_ir.bin");
|
||||
//! [part1]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <ngraph/ngraph.hpp>
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
using namespace ngraph;
|
||||
Core core;
|
||||
auto network = core.ReadNetwork("sample.xml");
|
||||
auto function = network.getFunction();
|
||||
//! [part0]
|
||||
for (auto && op : function->get_ops())
|
||||
op->get_rt_info()["affinity"] = std::shared_ptr<ngraph::VariantWrapper<std::string>>("CPU");
|
||||
//! [part0]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <ngraph/ngraph.hpp>
|
||||
#include <ngraph/function.hpp>
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
using namespace ngraph;
|
||||
//! [part1]
|
||||
InferenceEngine::Core core;
|
||||
auto network = core.ReadNetwork("sample.xml");
|
||||
auto function = network.getFunction();
|
||||
|
||||
// This example demonstrates how to perform default affinity initialization and then
|
||||
// correct affinity manually for some layers
|
||||
const std::string device = "HETERO:FPGA,CPU";
|
||||
|
||||
// QueryNetworkResult object contains map layer -> device
|
||||
InferenceEngine::QueryNetworkResult res = core.QueryNetwork(network, device, { });
|
||||
|
||||
// update default affinities
|
||||
res.supportedLayersMap["layerName"] = "CPU";
|
||||
|
||||
// set affinities to network
|
||||
for (auto&& node : function->get_ops()) {
|
||||
auto& affinity = res.supportedLayersMap[node->get_friendly_name()];
|
||||
// Store affinity mapping using node runtime information
|
||||
node->get_rt_info()["affinity"] = std::make_shared<ngraph::VariantWrapper<std::string>>(affinity);
|
||||
}
|
||||
|
||||
// load network with affinities set before
|
||||
auto executable_network = core.LoadNetwork(network, device);
|
||||
//! [part1]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#include <inference_engine.hpp>
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part2]
|
||||
InferenceEngine::Core core;
|
||||
auto network = core.ReadNetwork("sample.xml");
|
||||
auto executable_network = core.LoadNetwork(network, "HETERO:FPGA,CPU");
|
||||
//! [part2]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include "ie_plugin_config.hpp"
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part3]
|
||||
using namespace InferenceEngine::PluginConfigParams;
|
||||
using namespace InferenceEngine::HeteroConfigParams;
|
||||
|
||||
// ...
|
||||
InferenceEngine::Core core;
|
||||
core.SetConfig({ { KEY_HETERO_DUMP_GRAPH_DOT, YES } }, "HETERO");
|
||||
//! [part3]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
#include <inference_engine.hpp>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part0]
|
||||
InferenceEngine::Core core;
|
||||
std::vector<std::string> availableDevices = core.GetAvailableDevices();
|
||||
//! [part0]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <ie_plugin_config.hpp>
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part1]
|
||||
InferenceEngine::Core core;
|
||||
bool dumpDotFile = core.GetConfig("HETERO", HETERO_CONFIG_KEY(DUMP_GRAPH_DOT)).as<bool>();
|
||||
//! [part1]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
#include <inference_engine.hpp>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part2]
|
||||
InferenceEngine::Core core;
|
||||
std::string cpuDeviceName = core.GetMetric("GPU", METRIC_KEY(FULL_DEVICE_NAME)).as<std::string>();
|
||||
//! [part2]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#include <inference_engine.hpp>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part3]
|
||||
InferenceEngine::Core core;
|
||||
auto network = core.ReadNetwork("sample.xml");
|
||||
auto exeNetwork = core.LoadNetwork(network, "CPU");
|
||||
auto nireq = exeNetwork.GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<unsigned int>();
|
||||
//! [part3]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#include <inference_engine.hpp>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part4]
|
||||
InferenceEngine::Core core;
|
||||
auto network = core.ReadNetwork("sample.xml");
|
||||
auto exeNetwork = core.LoadNetwork(network, "MYRIAD");
|
||||
float temperature = exeNetwork.GetMetric(METRIC_KEY(DEVICE_THERMAL)).as<float>();
|
||||
//! [part4]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#include <inference_engine.hpp>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part5]
|
||||
InferenceEngine::Core core;
|
||||
auto network = core.ReadNetwork("sample.xml");
|
||||
auto exeNetwork = core.LoadNetwork(network, "CPU");
|
||||
auto ncores = exeNetwork.GetConfig(PluginConfigParams::KEY_CPU_THREADS_NUM).as<std::string>();
|
||||
//! [part5]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
#include <inference_engine.hpp>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
const std::string output_name = "output_name";
|
||||
const std::string input_name = "input_name";
|
||||
//! [part0]
|
||||
InferenceEngine::Core core;
|
||||
//! [part0]
|
||||
|
||||
//! [part1]
|
||||
auto network = core.ReadNetwork("Model.xml");
|
||||
//! [part1]
|
||||
|
||||
//! [part2]
|
||||
auto network = core.ReadNetwork("model.onnx");
|
||||
//! [part2]
|
||||
|
||||
//! [part3]
|
||||
/** Take information about all topology inputs **/
|
||||
InferenceEngine::InputsDataMap input_info = network.getInputsInfo();
|
||||
/** Take information about all topology outputs **/
|
||||
InferenceEngine::OutputsDataMap output_info = network.getOutputsInfo();
|
||||
//! [part3]
|
||||
|
||||
//! [part4]
|
||||
/** Iterate over all input info**/
|
||||
for (auto &item : input_info) {
|
||||
auto input_data = item.second;
|
||||
input_data->setPrecision(Precision::U8);
|
||||
input_data->setLayout(Layout::NCHW);
|
||||
input_data->getPreProcess().setResizeAlgorithm(RESIZE_BILINEAR);
|
||||
input_data->getPreProcess().setColorFormat(ColorFormat::RGB);
|
||||
}
|
||||
/** Iterate over all output info**/
|
||||
for (auto &item : output_info) {
|
||||
auto output_data = item.second;
|
||||
output_data->setPrecision(Precision::FP32);
|
||||
output_data->setLayout(Layout::NC);
|
||||
}
|
||||
//! [part4]
|
||||
|
||||
//! [part5]
|
||||
auto executable_network = core.LoadNetwork(network, "CPU");
|
||||
//! [part5]
|
||||
|
||||
//! [part6]
|
||||
/** Optional config. E.g. this enables profiling of performance counters. **/
|
||||
std::map<std::string, std::string> config = {{ PluginConfigParams::KEY_PERF_COUNT, PluginConfigParams::YES }};
|
||||
auto executable_network = core.LoadNetwork(network, "CPU", config);
|
||||
//! [part6]
|
||||
|
||||
//! [part7]
|
||||
auto infer_request = executable_network.CreateInferRequest();
|
||||
//! [part7]
|
||||
|
||||
auto infer_request1 = executable_network.CreateInferRequest();
|
||||
auto infer_request2 = executable_network.CreateInferRequest();
|
||||
|
||||
//! [part8]
|
||||
/** Iterate over all input blobs **/
|
||||
for (auto & item : input_info) {
|
||||
auto input_name = item.first;
|
||||
/** Get input blob **/
|
||||
auto input = infer_request.GetBlob(input_name);
|
||||
/** Fill input tensor with planes. First b channel, then g and r channels **/
|
||||
// ...
|
||||
}
|
||||
//! [part8]
|
||||
|
||||
//! [part9]
|
||||
auto output = infer_request1.GetBlob(output_name);
|
||||
infer_request2.SetBlob(input_name, output);
|
||||
//! [part9]
|
||||
|
||||
//! [part10]
|
||||
/** inputBlob points to input of a previous network and
|
||||
cropROI contains coordinates of output bounding box **/
|
||||
InferenceEngine::Blob::Ptr inputBlob;
|
||||
InferenceEngine::ROI cropRoi;
|
||||
//...
|
||||
|
||||
/** roiBlob uses shared memory of inputBlob and describes cropROI
|
||||
according to its coordinates **/
|
||||
auto roiBlob = InferenceEngine::make_shared_blob(inputBlob, cropRoi);
|
||||
infer_request2.SetBlob(input_name, roiBlob);
|
||||
//! [part10]
|
||||
|
||||
//! [part11]
|
||||
/** Iterate over all input blobs **/
|
||||
for (auto & item : input_info) {
|
||||
auto input_data = item.second;
|
||||
/** Create input blob **/
|
||||
InferenceEngine::TBlob<unsigned char>::Ptr input;
|
||||
// assuming input precision was asked to be U8 in prev step
|
||||
input = InferenceEngine::make_shared_blob<unsigned char, InferenceEngine::SizeVector>(InferenceEngine::Precision::U8, input_data->getDims());
|
||||
input->allocate();
|
||||
infer_request.SetBlob(item.first, input);
|
||||
|
||||
/** Fill input tensor with planes. First b channel, then g and r channels **/
|
||||
// ...
|
||||
}
|
||||
//! [part11]
|
||||
|
||||
//! [part12]
|
||||
infer_request.StartAsync();
|
||||
infer_request.Wait(IInferRequest::WaitMode::RESULT_READY);
|
||||
//! [part12]
|
||||
|
||||
auto sync_infer_request = executable_network.CreateInferRequest();
|
||||
|
||||
//! [part13]
|
||||
sync_infer_request.Infer();
|
||||
//! [part13]
|
||||
|
||||
//! [part14]
|
||||
for (auto &item : output_info) {
|
||||
auto output_name = item.first;
|
||||
auto output = infer_request.GetBlob(output_name);
|
||||
{
|
||||
auto const memLocker = output->cbuffer(); // use const memory locker
|
||||
// output_buffer is valid as long as the lifetime of memLocker
|
||||
const float *output_buffer = memLocker.as<const float *>();
|
||||
/** output_buffer[] - accessing output blob data **/
|
||||
}
|
||||
}
|
||||
//! [part14]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <multi-device/multi_device_config.hpp>
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part0]
|
||||
Core ie;
|
||||
auto network = ie.ReadNetwork("sample.xml");
|
||||
//NEW IE-CENTRIC API, the "MULTI" plugin is (globally) pre-configured with the explicit option:
|
||||
ie.SetConfig({{"MULTI_DEVICE_PRIORITIES", "HDDL,GPU"}}, "MULTI");
|
||||
ExecutableNetwork exec0 = ie.LoadNetwork(network, "MULTI", {});
|
||||
|
||||
//NEW IE-CENTRIC API, configuration of the "MULTI" is part of the network configuration (and hence specific to the network):
|
||||
ExecutableNetwork exec1 = ie.LoadNetwork(network, "MULTI", {{"MULTI_DEVICE_PRIORITIES", "HDDL,GPU"}});
|
||||
//NEW IE-CENTRIC API, same as previous, but configuration of the "MULTI" is part of the name (so config is empty), also network-specific:
|
||||
ExecutableNetwork exec2 = ie.LoadNetwork(network, "MULTI:HDDL,GPU", {});
|
||||
//! [part0]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <multi-device/multi_device_config.hpp>
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part1]
|
||||
Core ie;
|
||||
auto network = ie.ReadNetwork("sample.xml");
|
||||
ExecutableNetwork exec = ie.LoadNetwork(network, "MULTI:HDDL,GPU", {});
|
||||
//...
|
||||
exec.SetConfig({{"MULTI_DEVICE_PRIORITIES", "GPU,HDDL"}});
|
||||
// you can even exclude some device
|
||||
exec.SetConfig({{"MULTI_DEVICE_PRIORITIES", "GPU"}});
|
||||
//...
|
||||
// and then return it back
|
||||
exec.SetConfig({{"MULTI_DEVICE_PRIORITIES", "GPU,HDDL"}});
|
||||
//but you cannot add new devices on the fly, the next line will trigger the following exception:
|
||||
//[ ERROR ] [NOT_FOUND] You can only change device priorities but not add new devices with the Network's SetConfig(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES.
|
||||
//CPU device was not in the original device list!
|
||||
exec.SetConfig({{"MULTI_DEVICE_PRIORITIES", "CPU,GPU,HDDL"}});
|
||||
//! [part1]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <multi-device/multi_device_config.hpp>
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part2]
|
||||
Core ie;
|
||||
auto cnnNetwork = ie.ReadNetwork("sample.xml");
|
||||
std::string allDevices = "MULTI:";
|
||||
std::vector<std::string> availableDevices = ie.GetAvailableDevices();
|
||||
for (auto && device : availableDevices) {
|
||||
allDevices += device;
|
||||
allDevices += ((device == availableDevices[availableDevices.size()-1]) ? "" : ",");
|
||||
}
|
||||
ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, allDevices, {});
|
||||
//! [part2]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <multi-device/multi_device_config.hpp>
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part3]
|
||||
Core ie;
|
||||
auto cnnNetwork = ie.ReadNetwork("sample.xml");
|
||||
std::string allDevices = "MULTI:";
|
||||
std::vector<std::string> myriadDevices = ie.GetMetric("MYRIAD", METRIC_KEY(myriadDevices));
|
||||
for (int i = 0; i < myriadDevices.size(); ++i) {
|
||||
allDevices += std::string("MYRIAD.")
|
||||
+ myriadDevices[i]
|
||||
+ std::string(i < (myriadDevices.size() -1) ? "," : "");
|
||||
}
|
||||
ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, allDevices, {});
|
||||
//! [part3]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <multi-device/multi_device_config.hpp>
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
const std::map<std::string, std::string> hddl_config = { { PluginConfigParams::KEY_PERF_COUNT, PluginConfigParams::YES } };
|
||||
const std::map<std::string, std::string> gpu_config = { { PluginConfigParams::KEY_PERF_COUNT, PluginConfigParams::YES } };
|
||||
//! [part4]
|
||||
// configure the HDDL device first
|
||||
Core ie;
|
||||
CNNNetwork cnnNetwork = ie.ReadNetwork("sample.xml");
|
||||
ie.SetConfig(hddl_config, "HDDL");
|
||||
// configure the GPU device
|
||||
ie.SetConfig(gpu_config, "GPU");
|
||||
// load the network to the multi-device, while specifying the configuration (devices along with priorities):
|
||||
ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, "MULTI", {{MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, "HDDL,GPU"}});
|
||||
// new metric allows to query the optimal number of requests:
|
||||
uint32_t nireq = exeNetwork.GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<unsigned int>();
|
||||
//! [part4]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <multi-device/multi_device_config.hpp>
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
std::string device_name = "MULTI:HDDL,GPU";
|
||||
const std::map< std::string, std::string > full_config = {};
|
||||
//! [part5]
|
||||
Core ie;
|
||||
CNNNetwork cnnNetwork = ie.ReadNetwork("sample.xml");
|
||||
// 'device_name' can be "MULTI:HDDL,GPU" to configure the multi-device to use HDDL and GPU
|
||||
ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, device_name, full_config);
|
||||
// new metric allows to query the optimal number of requests:
|
||||
uint32_t nireq = exeNetwork.GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<unsigned int>();
|
||||
//! [part5]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <ie_cnn_network.h>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
std::string deviceName = "Device name";
|
||||
//! [part0]
|
||||
InferenceEngine::InferencePlugin plugin = InferenceEngine::PluginDispatcher({ FLAGS_pp }).getPluginByDevice(FLAGS_d);
|
||||
//! [part0]
|
||||
|
||||
//! [part1]
|
||||
InferenceEngine::Core core;
|
||||
//! [part1]
|
||||
|
||||
//! [part2]
|
||||
CNNNetReader network_reader;
|
||||
network_reader.ReadNetwork(fileNameToString(input_model));
|
||||
network_reader.ReadWeights(fileNameToString(input_model).substr(0, input_model.size() - 4) + ".bin");
|
||||
CNNNetwork network = network_reader.getNetwork();
|
||||
//! [part2]
|
||||
|
||||
//! [part3]
|
||||
CNNNetwork network = core.ReadNetwork(input_model);
|
||||
//! [part3]
|
||||
|
||||
//! [part4]
|
||||
CNNNetwork network = core.ReadNetwork("model.onnx");
|
||||
//! [part4]
|
||||
|
||||
//! [part5]
|
||||
plugin.AddExtension(std::make_shared<Extensions::Cpu::CpuExtensions>());
|
||||
//! [part5]
|
||||
|
||||
//! [part6]
|
||||
core.AddExtension(std::make_shared<Extensions::Cpu::CpuExtensions>(), "CPU");
|
||||
//! [part6]
|
||||
|
||||
//! [part7]
|
||||
core.SetConfig({{PluginConfigParams::KEY_CONFIG_FILE, FLAGS_c}}, "GPU");
|
||||
//! [part7]
|
||||
|
||||
//! [part8]
|
||||
auto execNetwork = plugin.LoadNetwork(network, { });
|
||||
//! [part8]
|
||||
|
||||
//! [part9]
|
||||
auto execNetwork = core.LoadNetwork(network, deviceName, { });
|
||||
//! [part9]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <ngraph/ngraph.hpp>
|
||||
#include "onnx/onnx-ml.pb.h"
|
||||
#include <iostream>
|
||||
#include <set>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part0]
|
||||
const std::int64_t version = 12;
|
||||
const std::string domain = "ai.onnx";
|
||||
const std::set<std::string> supported_ops = ngraph::onnx_import::get_supported_operators(version, domain);
|
||||
|
||||
for(const auto& op : supported_ops)
|
||||
{
|
||||
std::cout << op << std::endl;
|
||||
}
|
||||
//! [part0]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <ngraph/ngraph.hpp>
|
||||
#include "ngraph/frontend/onnx_import/onnx_utils.hpp"
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part1]
|
||||
const std::string op_name = "Abs";
|
||||
const std::int64_t version = 12;
|
||||
const std::string domain = "ai.onnx";
|
||||
const bool is_abs_op_supported = ngraph::onnx_import::is_operator_supported(op_name, version, domain);
|
||||
|
||||
std::cout << "Abs in version 12, domain `ai.onnx`is supported: " << (is_abs_op_supported ? "true" : "false") << std::endl;
|
||||
//! [part1]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <ngraph/ngraph.hpp>
|
||||
#include "ngraph/frontend/onnx_import/onnx.hpp"
|
||||
#include <iostream>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
using namespace ngraph;
|
||||
//! [part2]
|
||||
const std::string resnet50_path = "resnet50/model.onnx";
|
||||
std::ifstream resnet50_stream(resnet50_path);
|
||||
if(resnet50_stream.is_open())
|
||||
{
|
||||
try
|
||||
{
|
||||
const std::shared_ptr<ngraph::Function> ng_function = ngraph::onnx_import::import_onnx_model(resnet50_stream);
|
||||
|
||||
// Check shape of the first output, for example
|
||||
std::cout << ng_function->get_output_shape(0) << std::endl;
|
||||
// The output is Shape{1, 1000}
|
||||
}
|
||||
catch (const ngraph::ngraph_error& error)
|
||||
{
|
||||
std::cout << "Error when importing ONNX model: " << error.what() << std::endl;
|
||||
}
|
||||
}
|
||||
resnet50_stream.close();
|
||||
//! [part2]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <ngraph/ngraph.hpp>
|
||||
#include "ngraph/frontend/onnx_import/onnx.hpp"
|
||||
#include <iostream>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
using namespace ngraph;
|
||||
//! [part3]
|
||||
const std::shared_ptr<ngraph::Function> ng_function = ngraph::onnx_import::import_onnx_model(resnet50_path);
|
||||
//! [part3]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <opencv2/core.hpp>
|
||||
#include <opencv2/imgcodecs.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
using namespace cv;
|
||||
int batch_size = 1;
|
||||
//! [part0]
|
||||
InferenceEngine::Core core;
|
||||
// ------------- 0. Read IR and image ----------------------------------------------
|
||||
CNNNetwork network = core.ReadNetwork("path/to/IR/xml");
|
||||
cv::Mat image = cv::imread("path/to/image");
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
// ------------- 1. Collect the map of input names and shapes from IR---------------
|
||||
auto input_shapes = network.getInputShapes();
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
// ------------- 2. Set new input shapes -------------------------------------------
|
||||
std::string input_name;
|
||||
SizeVector input_shape;
|
||||
std::tie(input_name, input_shape) = *input_shapes.begin(); // let's consider first input only
|
||||
input_shape[0] = batch_size; // set batch size to the first input dimension
|
||||
input_shape[2] = image.rows; // changes input height to the image one
|
||||
input_shape[3] = image.cols; // changes input width to the image one
|
||||
input_shapes[input_name] = input_shape;
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
// ------------- 3. Call reshape ---------------------------------------------------
|
||||
network.reshape(input_shapes);
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
//...
|
||||
|
||||
// ------------- 4. Loading model to the device ------------------------------------
|
||||
std::string device = "CPU";
|
||||
ExecutableNetwork executable_network = core.LoadNetwork(network, device);
|
||||
// ---------------------------------------------------------------------------------
|
||||
|
||||
|
||||
//! [part0]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include "ie_plugin_config.hpp"
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part0]
|
||||
using namespace InferenceEngine::PluginConfigParams;
|
||||
using namespace InferenceEngine::HeteroConfigParams;
|
||||
|
||||
Core ie;
|
||||
auto network = ie.ReadNetwork("sample.xml");
|
||||
// ...
|
||||
|
||||
auto execNetwork = ie.LoadNetwork(network, "HETERO:FPGA,CPU", { {KEY_HETERO_DUMP_GRAPH_DOT, YES} });
|
||||
//! [part0]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include "ie_plugin_config.hpp"
|
||||
#include <ie_input_info.hpp>
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part1]
|
||||
Core ie;
|
||||
auto netReader = ie.ReadNetwork("sample.xml");
|
||||
InferenceEngine::InputsDataMap info(netReader.getInputsInfo());
|
||||
auto& inputInfoFirst = info.begin()->second;
|
||||
for (auto& it : info) {
|
||||
it.second->setPrecision(Precision::U8);
|
||||
}
|
||||
//! [part1]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include "ie_plugin_config.hpp"
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part2]
|
||||
//Lock Intel MSS surface
|
||||
mfxFrameSurface1 *frame_in; //Input MSS surface.
|
||||
mfxFrameAllocator* pAlloc = &m_mfxCore.FrameAllocator();
|
||||
pAlloc->Lock(pAlloc->pthis, frame_in->Data.MemId, &frame_in->Data);
|
||||
//Inference Engine code
|
||||
//! [part2]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include "ie_plugin_config.hpp"
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part3]
|
||||
InferenceEngine::SizeVector dims_src = {
|
||||
1 /* batch, N*/,
|
||||
(size_t) frame_in->Info.Height /* Height */,
|
||||
(size_t) frame_in->Info.Width /* Width */,
|
||||
3 /*Channels,*/,
|
||||
};
|
||||
TensorDesc desc(InferenceEngine::Precision::U8, dims_src, InferenceEngine::NHWC);
|
||||
/* wrapping the surface data, as RGB is interleaved, need to pass only ptr to the R, notice that this wouldn’t work with planar formats as these are 3 separate planes/pointers*/
|
||||
InferenceEngine::TBlob<uint8_t>::Ptr p = InferenceEngine::make_shared_blob<uint8_t>( desc, (uint8_t*) frame_in->Data.R);
|
||||
inferRequest.SetBlob("input", p);
|
||||
inferRequest.Infer();
|
||||
//Make sure to unlock the surface upon inference completion, to return the ownership back to the Intel MSS
|
||||
pAlloc->Unlock(pAlloc->pthis, frame_in->Data.MemId, &frame_in->Data);
|
||||
//! [part3]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include "ie_plugin_config.hpp"
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part4]
|
||||
InferenceEngine::SizeVector dims_src = {
|
||||
1 /* batch, N*/,
|
||||
3 /*Channels,*/,
|
||||
(size_t) frame_in->Info.Height /* Height */,
|
||||
(size_t) frame_in->Info.Width /* Width */,
|
||||
};
|
||||
TensorDesc desc(InferenceEngine::Precision::U8, dims_src, InferenceEngine::NCHW);
|
||||
/* wrapping the RGBP surface data*/
|
||||
InferenceEngine::TBlob<uint8_t>::Ptr p = InferenceEngine::make_shared_blob<uint8_t>( desc, (uint8_t*) frame_in->Data.R);
|
||||
inferRequest.SetBlob("input", p);
|
||||
// …
|
||||
//! [part4]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include <opencv2/core/core.hpp>
|
||||
#include "ie_plugin_config.hpp"
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part5]
|
||||
cv::Mat frame = ...; // regular CV_8UC3 image, interleaved
|
||||
// creating blob that wraps the OpenCV’s Mat
|
||||
// (the data it points should persists until the blob is released):
|
||||
InferenceEngine::SizeVector dims_src = {
|
||||
1 /* batch, N*/,
|
||||
(size_t)frame.rows /* Height */,
|
||||
(size_t)frame.cols /* Width */,
|
||||
(size_t)frame.channels() /*Channels,*/,
|
||||
};
|
||||
TensorDesc desc(InferenceEngine::Precision::U8, dims_src, InferenceEngine::NHWC);
|
||||
InferenceEngine::TBlob<uint8_t>::Ptr p = InferenceEngine::make_shared_blob<uint8_t>( desc, (uint8_t*)frame.data, frame.step[0] * frame.rows);
|
||||
inferRequest.SetBlob("input", p);
|
||||
inferRequest.Infer();
|
||||
// …
|
||||
// similarly, you can wrap the output tensor (let’s assume it is FP32)
|
||||
// notice that the output should be also explicitly stated as NHWC with setLayout
|
||||
const float* output_data = output_blob->buffer().
|
||||
as<PrecisionTrait<Precision::FP32>::value_type*>();
|
||||
cv::Mat res (rows, cols, CV_32FC3, output_data, CV_AUTOSTEP);
|
||||
//! [part5]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include "ie_plugin_config.hpp"
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part6]
|
||||
Core ie;
|
||||
auto network = ie.ReadNetwork("Model.xml", "Model.bin");
|
||||
InferenceEngine::InputsDataMap input_info(network.getInputsInfo());
|
||||
|
||||
|
||||
auto executable_network = ie.LoadNetwork(network, "GPU");
|
||||
auto infer_request = executable_network.CreateInferRequest();
|
||||
|
||||
for (auto & item : input_info) {
|
||||
std::string input_name = item.first;
|
||||
auto input = infer_request.GetBlob(input_name);
|
||||
/** Lock/Fill input tensor with data **/
|
||||
unsigned char* data = input->buffer().as<PrecisionTrait<Precision::U8>::value_type*>();
|
||||
// ...
|
||||
}
|
||||
|
||||
|
||||
infer_request.Infer();
|
||||
//! [part6]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include "ie_plugin_config.hpp"
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
Core plugin;
|
||||
auto network0 = plugin.ReadNetwork("sample.xml");
|
||||
auto network1 = plugin.ReadNetwork("sample.xml");
|
||||
//! [part7]
|
||||
//these two networks go thru same plugin (aka device) and their requests will not overlap.
|
||||
auto executable_network0 = plugin.LoadNetwork(network0, "CPU", {{PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, PluginConfigParams::YES}});
|
||||
auto executable_network1 = plugin.LoadNetwork(network1, "GPU", {{PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, PluginConfigParams::YES}});
|
||||
//! [part7]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include "ie_plugin_config.hpp"
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part8]
|
||||
while(…) {
|
||||
capture frame
|
||||
populate CURRENT InferRequest
|
||||
Infer CURRENT InferRequest //this call is synchronous
|
||||
display CURRENT result
|
||||
}
|
||||
//! [part8]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include "ie_plugin_config.hpp"
|
||||
#include "hetero/hetero_plugin_config.hpp"
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part9]
|
||||
while(…) {
|
||||
capture frame
|
||||
populate NEXT InferRequest
|
||||
start NEXT InferRequest //this call is async and returns immediately
|
||||
wait for the CURRENT InferRequest //processed in a dedicated thread
|
||||
display CURRENT result
|
||||
swap CURRENT and NEXT InferRequests
|
||||
}
|
||||
//! [part9]
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
#include <inference_engine.hpp>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
Core core;
|
||||
int numRequests = 42;
|
||||
int i = 1;
|
||||
auto network = core.ReadNetwork("sample.xml");
|
||||
auto executable_network = core.LoadNetwork(network, "CPU");
|
||||
//! [part0]
|
||||
struct Request {
|
||||
InferenceEngine::InferRequest::Ptr inferRequest;
|
||||
int frameidx;
|
||||
};
|
||||
//! [part0]
|
||||
|
||||
//! [part1]
|
||||
// numRequests is the number of frames (max size, equal to the number of VPUs in use)
|
||||
std::vector<Request> request(numRequests);
|
||||
//! [part1]
|
||||
|
||||
//! [part2]
|
||||
// initialize infer request pointer – Consult IE API for more detail.
|
||||
request[i].inferRequest = executable_network.CreateInferRequestPtr();
|
||||
//! [part2]
|
||||
|
||||
//! [part3]
|
||||
// Run inference
|
||||
request[i].inferRequest->StartAsync();
|
||||
//! [part3]
|
||||
|
||||
//! [part4]
|
||||
request[i].inferRequest->SetCompletionCallback(InferenceEngine::IInferRequest::Ptr context);
|
||||
//! [part4]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
#include <inference_engine.hpp>
|
||||
#include "ngraph/opsets/opset.hpp"
|
||||
#include "ngraph/opsets/opset3.hpp"
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part0]
|
||||
|
||||
using namespace std;
|
||||
using namespace ngraph;
|
||||
|
||||
auto arg0 = make_shared<opset3::Parameter>(element::f32, Shape{7});
|
||||
auto arg1 = make_shared<opset3::Parameter>(element::f32, Shape{7});
|
||||
// Create an 'Add' operation with two inputs 'arg0' and 'arg1'
|
||||
auto add0 = make_shared<opset3::Add>(arg0, arg1);
|
||||
auto abs0 = make_shared<opset3::Abs>(add0);
|
||||
// Create a node whose inputs/attributes will be specified later
|
||||
auto acos0 = make_shared<opset3::Acos>();
|
||||
// Create a node using opset factories
|
||||
auto add1 = shared_ptr<Node>(get_opset3().create("Add"));
|
||||
// Set inputs to nodes explicitly
|
||||
acos0->set_argument(0, add0);
|
||||
add1->set_argument(0, acos0);
|
||||
add1->set_argument(1, abs0);
|
||||
|
||||
// Run shape inference on the nodes
|
||||
NodeVector ops{arg0, arg1, add0, abs0, acos0, add1};
|
||||
validate_nodes_and_infer_types(ops);
|
||||
|
||||
// Create a graph with one output (add1) and four inputs (arg0, arg1)
|
||||
auto ng_function = make_shared<Function>(OutputVector{add1}, ParameterVector{arg0, arg1});
|
||||
|
||||
//! [part0]
|
||||
|
||||
//! [part1]
|
||||
CNNNetwork net (ng_function);
|
||||
//! [part1]
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
#include <inference_engine.hpp>
|
||||
|
||||
int main() {
|
||||
using namespace InferenceEngine;
|
||||
//! [part0]
|
||||
std::vector<uint8_t> model;
|
||||
std::vector<uint8_t> weights;
|
||||
|
||||
// Read model files and decrypt them into temporary memory block
|
||||
decrypt_file(model_file, password, model);
|
||||
decrypt_file(weights_file, password, weights);
|
||||
//! [part0]
|
||||
|
||||
//! [part1]
|
||||
Core core;
|
||||
// Load model from temporary memory block
|
||||
std::string strModel(model.begin(), model.end());
|
||||
CNNNetwork network = core.ReadNetwork(strModel, make_shared_blob<uint8_t>({Precision::U8, {weights.size()}, C}, weights.data()));
|
||||
//! [part1]
|
||||
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user