IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated documentation for OV Inference Runtime (#9573)

Browse Source 
* Updated documentation for OV Inference Runtime

* Updated documentation

* Fixed comments
This commit is contained in:
Ilya Lavrenov 2022-01-13 01:05:17 +03:00 committed by GitHub
parent 00da13b058
commit f222814f0c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 354 additions and 283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/core/include/openvino/core/any.hpp
View File

@ -375,7 +375,7 @@ public:
Any& operator=(Any&& other) = default;
/**
* @brief Destructor presereves unload order of implementation object and reference to library
* @brief Destructor preserves unloading order of implementation object and reference to library
*/
~Any();

2
src/core/include/openvino/runtime/allocator.hpp
View File

@ -79,7 +79,7 @@ class OPENVINO_API Allocator {
public:
/**
* @brief Destructor presereves unload order of implementation object and reference to library
* @brief Destructor preserves unloading order of implementation object and reference to library
*/
~Allocator();

2
src/core/include/openvino/runtime/tensor.hpp
View File

@ -75,7 +75,7 @@ public:
Tensor& operator=(Tensor&& other) = default;
/**
* @brief Destructor presereves unload order of implementation object and reference to library
* @brief Destructor preserves unloading order of implementation object and reference to library
*/
~Tensor();

2
src/inference/dev_api/so_ptr.hpp
View File

@ -31,7 +31,7 @@ struct SoPtr {
SoPtr() = default;
/**
* @brief Destructor presereves unload order of implementation object and reference to library
* @brief Destructor preserves unloading order of implementation object and reference to library
*/
~SoPtr() {
_ptr = {};

2
src/inference/include/ie/cpp/ie_executable_network.hpp
View File

@ -71,7 +71,7 @@ public:
ExecutableNetwork& operator=(ExecutableNetwork&& other) = default;
/**
* @brief Destructor presereves unload order of implementation object and reference to library
* @brief Destructor preserves unloading order of implementation object and reference to library
*/
~ExecutableNetwork();

2
src/inference/include/ie/cpp/ie_infer_request.hpp
View File

@ -83,7 +83,7 @@ public:
InferRequest& operator=(InferRequest&& other) = default;
/**
* @brief Destructor presereves unload order of implementation object and reference to library
* @brief Destructor preserves unloading order of implementation object and reference to library
*/
~InferRequest();

2
src/inference/include/ie/cpp/ie_memory_state.hpp
View File

@ -60,7 +60,7 @@ public:
VariableState& operator=(VariableState&& other) = default;
/**
* @brief Destructor presereves unload order of implementation object and reference to library
* @brief Destructor preserves unloading order of implementation object and reference to library
*/
~VariableState();

2
src/inference/include/openvino/runtime/common.hpp
View File

@ -45,7 +45,7 @@ namespace ov {
namespace ie = InferenceEngine;
namespace runtime {
/**
* @brief This type of map is commonly used to pass set of parameters
* @brief This type of map is commonly used to pass set of device configuration parameters
*/
using ConfigMap = std::map<std::string, std::string>;

115
src/inference/include/openvino/runtime/compiled_model.hpp
View File

@ -31,7 +31,9 @@ class Core;
class InferRequest;
/**
* @brief This is an interface of an executable network
* @brief This class represents compiled model
* Model is compiled by a specific device by applying multiple optimization
* transformations, then mapping to compute kernels.
*/
class OPENVINO_RUNTIME_API CompiledModel {
std::shared_ptr<InferenceEngine::IExecutableNetworkInternal> _impl;
@ -55,128 +57,139 @@ public:
CompiledModel() = default;
/**
* @brief Destructor presereves unload order of implementation object and reference to library
* @brief Destructor preserves unloading order of implementation object and reference to library
*/
~CompiledModel();
/**
* @brief Get executable graph information from a device
* @brief Get executable model information from a device
* This object represents the internal device specific model which is optimized for particular
* accelerator. It contains device specific nodes, runtime information and can be used only
* to understand how the source model is optimized and which kernels, element types and layouts
* are selected for optimal inference.
*
* @return Model containing Executable Graph Info
*/
std::shared_ptr<const Model> get_runtime_model() const;
/**
* @brief Get inputs of executable graph
*
* @return vector of inputs
* @brief Gets all inputs of a compiled model
* Inputs are represented as a vector of outputs of ov::op::v0::Parameter operations.
* They contain information about input tensors such as tensor shape, names and element type
* @return std::vector of model inputs
*/
std::vector<ov::Output<const ov::Node>> inputs() const;
/**
* @brief Get input of executable graph
*
* @return Model input or throw ov::Exception in case of several outputs
* @brief Gets a single input of a compiled model
* An input is represented as an output of ov::op::v0::Parameter operation.
* An input contain information about input tensor such as tensor shape, names and element type
* @return A compiled model input
* @note If a model has more than one input, this method throws an ov::Exception
*/
ov::Output<const ov::Node> input() const;
/**
* @brief Get input of executable graph
*
* @param i input index
* @return Model input or throw ov::Exception if input wasn't found
* @brief Gets input of a compiled model identified by an @p i
* An input contains information about input tensor such as tensor shape, names and element type
* @param i An input index
* @return A compiled model input
* @note The method throws ov::Exception if input with specified index @p i is not found
*/
ov::Output<const ov::Node> input(size_t i) const;
/**
* @brief Get input of executable graph
*
* @brief Gets input of a compiled model identified by a @p tensor_name
* An input contain information about input tensor such as tensor shape, names and element type
* @param tensor_name The input tensor name
* @return Model output or throw ov::Exception if input wasn't found
* @return A compiled model input
* @note The method throws ov::Exception if input with specified tensor name @p tensor_name is not found
*/
ov::Output<const ov::Node> input(const std::string& tensor_name) const;
/**
* @brief Get outputs of executable graph model
*
* @return vector of outputs
* @brief Get all outputs of a compiled model
* Outputs are represented as a vector of output from ov::op::v0::Result operations.
* Outputs contain information about output tensors such as tensor shape, names and element type
* @return std::vector of model outputs
*/
std::vector<ov::Output<const ov::Node>> outputs() const;
/**
* @brief Get output of executable graph
*
* @return Model output or throw ov::Exception in case of several outputs
* @brief Gets a single output of a compiled model
* An output is represented as an output from ov::op::v0::Result operation.
* An output contain information about output tensor such as tensor shape, names and element type
* @return A compiled model output
* @note If a model has more than one output, this method throws an ov::Exception
*/
ov::Output<const ov::Node> output() const;
/**
* @brief Get output of executable graph
*
* @param i output index
* @return Model output or throw ov::Exception if output wasn't found
* @brief Gets output of a compiled model identified by an @p index
* An output contain information about output tensor such as tensor shape, names and element type
* @param i An output index
* @return A compiled model output
* @note The method throws ov::Exception if output with specified index @p index is not found
*/
ov::Output<const ov::Node> output(size_t i) const;
/**
* @brief Get output of executable graph
*
* @brief Gets output of a compiled model identified by a @p tensor_name
* An output contain information about output tensor such as tensor shape, names and element type
* @param tensor_name The output tensor name
* @return Model output or throw ov::Exception if output wasn't found
* @return A compiled model output
* @note The method throws ov::Exception if output with specified tensor name @p tensor_name is not found
*/
ov::Output<const ov::Node> output(const std::string& tensor_name) const;
/**
* @brief Creates an inference request object used to infer the model.
*
* The created request has allocated input and output blobs (that can be changed later).
* @brief Creates an inference request object used to infer the compiled model.
* The created request has allocated input and output tensors (that can be changed later).
*
* @return InferRequest object
*/
InferRequest create_infer_request();
/**
* @brief Exports the current executable network.
*
* @see Core::import_model
*
* @param model_stream Model output stream
* @brief Exports the current compiled model to an output stream `std::ostream`.
* The exported model can also be imported via ov::runtime::Core::import_model method
* @see ov::runtime::Core::import_model
* @param model_stream Output stream to store the model to
*/
void export_model(std::ostream& model_stream);
/**
* @brief Sets configuration for current executable network
*
* @brief Sets configuration for current compiled model
* @param config Map of pairs: (config parameter name, config parameter value)
*/
void set_config(const ParamMap& config);
/** @brief Gets configuration for current executable network.
/** @brief Gets configuration for a compiled model.
*
* The method is responsible to extract information
* which affects executable network execution. The list of supported configuration values can be extracted via
* which affects compiled model inference. The list of supported configuration values can be extracted via
* CompiledModel::get_metric with the SUPPORTED_CONFIG_KEYS key, but some of these keys cannot be changed
* dynamically, e.g. DEVICE_ID cannot changed if an executable network has already been compiled for particular
* dynamically, e.g. DEVICE_ID cannot changed if a compiled model has already been compiled for particular
* device.
*
* @param name config key, can be found in ie_plugin_config.hpp
* @param key_name config key, can be found in ie_plugin_config.hpp
* @return Configuration parameter value
*/
Any get_config(const std::string& name) const;
Any get_config(const std::string& key_name) const;
/**
* @brief Gets general runtime metric for an executable network.
* @brief Gets general runtime metric for a compiled model.
*
* It can be model name, actual device ID on
* which executable network is running or all other properties which cannot be changed dynamically.
* which compiled model is running or all other properties which cannot be changed dynamically.
*
* @param name metric name to request
* @param metric_name metric name to request
* @return Metric parameter value
*/
Any get_metric(const std::string& name) const;
Any get_metric(const std::string& metric_name) const;
/**
* @brief Returns pointer to plugin-specific shared context
* @brief Returns pointer to device-specific shared context
* on remote accelerator device that was used to create this CompiledModel
* @return A context
*/
@ -184,13 +197,13 @@ public:
/**
* @brief Checks if current CompiledModel object is not initialized
* @return true if current CompiledModel object is not initialized, false - otherwise
* @return `true` if current CompiledModel object is not initialized, `false` - otherwise
*/
bool operator!() const noexcept;
/**
* @brief Checks if current CompiledModel object is initialized
* @return true if current CompiledModel object is initialized, false - otherwise
* @return `true` if current CompiledModel object is initialized, `false` - otherwise
*/
explicit operator bool() const noexcept;
};

210
src/inference/include/openvino/runtime/core.hpp
View File

@ -36,8 +36,9 @@ namespace runtime {
/**
* @brief This class represents OpenVINO runtime Core entity.
*
* It can throw exceptions safely for the application, where it is properly handled.
* User applications can create several Core class instances, but in this case the underlying plugins
* are created multiple times and not shared between several Core instances. The recommended way is to have
* a single Core instance per application.
*/
class OPENVINO_RUNTIME_API Core {
class Impl;
@ -45,170 +46,180 @@ class OPENVINO_RUNTIME_API Core {
public:
/** @brief Constructs OpenVINO Core instance using XML configuration file with
* plugins description.
* devices and their plugins description.
*
* See register_plugins for more details.
* See Core::register_plugins for more details.
*
* @param xml_config_file A path to .xml file with plugins to load from. If XML configuration file is not specified,
* then default OpenVINO Runtime plugins are loaded from the default plugin.xml file.
* then default OpenVINO Runtime plugins are loaded from the default `plugin.xml` file located in the same folder
* as OpenVINO runtime shared library.
*/
explicit Core(const std::string& xml_config_file = {});
/**
* @brief Returns plugins version information
* @brief Returns device plugins version information
* Device name can be complex and identify multiple devices at once like `HETERO:CPU,GPU`
* and in this case a std::map contains multiple entries each per device.
*
* @param device_name Device name to identify plugin
* @param device_name Device name to identify a plugin
* @return A vector of versions
*/
std::map<std::string, Version> get_versions(const std::string& device_name) const;
#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
/**
* @brief Reads models from IR and ONNX formats
* @param model_path path to model
* @param bin_path path to data file
* @brief Reads models from IR / ONNX / PDPD formats
* @param model_path A path to a model
* @param bin_path A path to a data file
* For IR format (*.bin):
* * if path is empty, will try to read bin file with the same name as xml and
* * if bin file with the same name was not found, will load IR without weights.
* For ONNX format (*.onnx):
* * bin_path parameter is not used.
* @return Model
* For PDPD format (*.pdmodel)
* * bin_path parameter is not used.
* @return A model
*/
std::shared_ptr<ov::Model> read_model(const std::wstring& model_path, const std::wstring& bin_path = {}) const;
#endif
/**
* @brief Reads models from IR and ONNX formats
* @param model_path path to model
* @param bin_path path to data file
* @brief Reads models from IR / ONNX / PDPD formats
* @param model_path A path to a model
* @param bin_path A path to a data file
* For IR format (*.bin):
* * if path is empty, will try to read bin file with the same name as xml and
* * if bin file with the same name was not found, will load IR without weights.
* For ONNX format (*.onnx):
* * bin_path parameter is not used.
* @return Model
* For PDPD format (*.pdmodel)
* * bin_path parameter is not used.
* @return A model
*/
std::shared_ptr<ov::Model> read_model(const std::string& model_path, const std::string& bin_path = {}) const;
/**
* @brief Reads models from IR and ONNX formats
* @param model string with model in IR or ONNX format
* @param weights shared pointer to constant tensor with weights
* Reading ONNX models doesn't support loading weights from data tensors.
* @note Created model object shares the weights with `weights` object.
* So, do not create `weights` on temporary data which can be later freed, since the model
* constant data becomes to point to invalid memory.
* @return Model
* @brief Reads models from IR / ONNX / PDPD formats
* @param model A string with model in IR / ONNX / PDPD format
* @param weights A shared pointer to constant tensor with weights
* Reading ONNX / PDPD models doesn't support loading weights from @p weights tensors.
* @note Created model object shares the weights with @p weights object.
* So, do not create @p weights on temporary data which can be later freed, since the model
* constant data becomes point to an invalid memory.
* @return A model
*/
std::shared_ptr<ov::Model> read_model(const std::string& model, const Tensor& weights) const;
/**
* @brief Creates an executable network from a model object and loads model to default device.
* @brief Creates and loads a compiled model from a source model to the default OpenVINO device selected by AUTO
* plugin.
*
* Users can create as many executable networks as they need and use
* Users can create as many compiled models as they need and use
* them simultaneously (up to the limitation of the hardware resources)
*
* @param model Model object acquired from Core::read_model
* @param config Optional map of pairs: (config parameter name, config parameter value) relevant only for this load
* operation
* @return An executable network reference
* @return A compiled model
*/
CompiledModel compile_model(const std::shared_ptr<const ov::Model>& model, const ConfigMap& config = {});
/**
* @brief Creates an executable network from a model object.
* @brief Creates a compiled model from a source model object.
*
* Users can create as many executable networks as they need and use
* Users can create as many compiled models as they need and use
* them simultaneously (up to the limitation of the hardware resources)
*
* @param model Model object acquired from Core::read_model
* @param device_name Name of device to load model to
* @param config Optional map of pairs: (config parameter name, config parameter value) relevant only for this load
* operation
* @return An executable network reference
* @return A compiled model
*/
CompiledModel compile_model(const std::shared_ptr<const ov::Model>& model,
const std::string& device_name,
const ConfigMap& config = {});
/**
* @brief Reads model and creates an executable network from IR or ONNX file and load model to default device.
* @brief Reads and loads a compiled model from IR / ONNX / PDPD file to the default OpenVINI device selected by
* AUTO plugin.
*
* This can be more efficient than using read_model + compile_model(Model) flow
* This can be more efficient than using Core::read_model + Core::compile_model(model_in_memory_object) flow
* especially for cases when caching is enabled and cached model is available
*
* @param model_path path to model
* @param config Optional map of pairs: (config parameter name, config parameter value) relevant only for this load
* operation/
*
* @return An executable network reference
* @return A compiled model
*/
CompiledModel compile_model(const std::string& model_path, const ConfigMap& config = {});
/**
* @brief Reads model and creates an executable network from IR or ONNX file
* @brief Reads model and creates a compiled model from IR / ONNX / PDPD file
*
* This can be more efficient than using read_model + compile_model(Model) flow
* This can be more efficient than using Core::read_model + Core::compile_model(model_in_memory_object) flow
* especially for cases when caching is enabled and cached model is available
*
* @param model_path path to model
* @param device_name Name of device to load model to
* @param model_path Path to a model
* @param device_name Name of device to load a model to
* @param config Optional map of pairs: (config parameter name, config parameter value) relevant only for this load
* operation/
*
* @return An executable network reference
* @return A compiled model
*/
CompiledModel compile_model(const std::string& model_path,
const std::string& device_name,
const ConfigMap& config = {});
/**
* @brief Creates an executable network from a network object within a specified remote context.
* @brief Creates a compiled model from a source model within a specified remote context.
* @param model Model object acquired from Core::read_model
* @param context Pointer to RemoteContext object
* @param context A reference to a RemoteContext object
* @param config Optional map of pairs: (config parameter name, config parameter value) relevant only for this load
* operation
* @return An executable network object
* @return A compiled model object
*/
CompiledModel compile_model(const std::shared_ptr<const ov::Model>& model,
const RemoteContext& context,
const ConfigMap& config = {});
/**
* @brief Registers extension
* @deprecated This method is deprecated. Please use other add_extension methods
* @deprecated This method is deprecated. Please use other Core::add_extension methods
* @brief Registers OpenVINO 1.0 extension to a Core object
* @param extension Pointer to already loaded extension
*/
OPENVINO_DEPRECATED("Please use add_extension(ov::Extension) or add_extension(path_to_library) instead.")
void add_extension(const std::shared_ptr<InferenceEngine::IExtension>& extension);
/**
* @brief Registers extension
* @param library_path path to library with ov::Extension
* @brief Registers an extension to a Core object
* @param library_path Path to library with ov::Extension
*/
void add_extension(const std::string& library_path);
#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
/**
* @brief Registers extension
* @param library_path path to library with ov::Extension
* @brief Registers an extension to a Core object
* @param library_path Unicode path to library with ov::Extension
*/
void add_extension(const std::wstring& library_path);
#endif
/**
* @brief Registers extension
* @param extension Pointer to base extension
* @brief Registers an extension to a Core object
* @param extension Pointer to extension
*/
void add_extension(const std::shared_ptr<ov::Extension>& extension);
/**
* @brief Registers extensions
* @param extensions Vector of loaded base extensions
* @brief Registers extensions to a Core object
* @param extensions Vector of loaded extensions
*/
void add_extension(const std::vector<std::shared_ptr<ov::Extension>>& extensions);
/**
* @brief Registers extension
* @brief Registers an extension to a Core object
* @param extension Extension class which is inherited from ov::Extension class
*/
template <class T, typename std::enable_if<std::is_base_of<ov::Extension, T>::value, bool>::type = true>
@ -218,9 +229,9 @@ public:
}
/**
* @brief Registers extensions
* @brief Registers extensions to a Core object
* @param extension Extension class which is inherited from ov::Extension class
* @param args list of extensions
* @param args A list of extensions
*/
template <class T,
class... Targs,
@ -232,7 +243,7 @@ public:
}
/**
* @brief Registers custom operation
* @brief Registers a custom operation inherited from ov::op::Op
*/
template <class T, typename std::enable_if<std::is_base_of<ov::op::Op, T>::value, bool>::type = true>
void add_extension() {
@ -241,7 +252,7 @@ public:
}
/**
* @brief Registers custom operations
* @brief Registers custom operations inherited from ov::op::Op
*/
template <class T,
class... Targs,
@ -254,26 +265,28 @@ public:
}
/**
* @brief Creates an executable network from a previously exported one
* @param model_stream Model stream
* @param device_name Name of device load executable network on
* @brief Imports a compiled model from a previously exported one
* @param model_stream std::istream input stream containing a model previously exported using
* ov::runtime::CompiledModel::export_model method
* @param device_name Name of device to import compiled model for. Note, if @p device_name device was not used to
* compile the original mode, an exception is thrown
* @param config Optional map of pairs: (config parameter name, config parameter value) relevant only for this load
* operation*
* @return An executable network reference
* @return A compiled model
*/
CompiledModel import_model(std::istream& model_stream,
const std::string& device_name,
const ConfigMap& config = {});
/**
* @brief Creates an executable network from a previously exported one within a specified
* remote context.
*
* @param model_stream Model stream
* @param context Pointer to RemoteContext object
* @brief Imports a compiled model from a previously exported one with a specified remote context.
* @param model_stream std::istream input stream containing a model previously exported from
* ov::runtime::CompiledModel::export_model
* @param context A reference to a RemoteContext object. Note, if the device from @p context was not used to compile
* the original mode, an exception is thrown
* @param config Optional map of pairs: (config parameter name, config parameter value) relevant only for this load
* operation
* @return An executable network reference
* @return A compiled model
*/
CompiledModel import_model(std::istream& model_stream, const RemoteContext& context, const ConfigMap& config = {});
@ -301,56 +314,62 @@ public:
/**
* @brief Gets configuration dedicated to device behaviour.
* The method is targeted to extract information which can be set via Core::set_config method.
*
* The method is targeted to extract information which can be set via set_config method.
*
* @param device_name - A name of a device to get a configuration value.
* @param name - config key.
* @param device_name A name of a device to get a configuration value.
* @param config_key_name A config key name.
* @return Value of config corresponding to config key.
*/
Any get_config(const std::string& device_name, const std::string& name) const;
Any get_config(const std::string& device_name, const std::string& config_key_name) const;
/**
* @brief Gets general runtime metric for dedicated hardware.
*
* The method is needed to request common device properties.
* The method is needed to request common device or system properties.
* It can be device name, temperature, other devices-specific values.
*
* @param device_name - A name of a device to get a metric value.
* @param name - metric name to request.
* @param device_name A name of a device to get a metric value.
* @param metric_key_name A metric name to request.
* @return Metric value corresponding to metric key.
*/
Any get_metric(const std::string& device_name, const std::string& name) const;
Any get_metric(const std::string& device_name, const std::string& metric_key_name) const;
/**
* @brief Returns devices available for inference
* Core objects goes over all registered plugins and asks about available devices.
*
* @return A vector of devices. The devices are returned as { CPU, GPU.0, GPU.1, MYRIAD }
* If there more than one device of specific type, they are enumerated with .# suffix.
* Such enumerated device can later be used as a device name in all Core methods like Core::compile_model,
* Core::query_model, Core::set_config and so on.
*/
std::vector<std::string> get_available_devices() const;
/**
* @brief Register new device and plugin which implement this device inside OpenVINO Runtime.
* @brief Register a new device and plugin which enable this device inside OpenVINO Runtime.
*
* @param plugin_name A name of plugin. Depending on platform `plugin_name` is wrapped with shared library suffix
* and prefix to identify library full name
* E.g. on Linux platform plugin name specified as `plugin_name` will be wrapped as `libplugin_name.so`.
* Plugin search algorithm:
* - If plugin is located in the same directory as OpenVINO runtime library, it will be used
* - If no, plugin is tried to be loaded from paths pointed by PATH / LD_LIBRARY_PATH / DYLD_LIBRARY_PATH
* environment variables depending on the platform.
*
* @param device_name A device name to register plugin for. If device name is not specified, then it's taken from
* plugin itself.
* @param device_name A device name to register plugin for.
*/
void register_plugin(const std::string& plugin_name, const std::string& device_name);
/**
* @brief Unloads previously loaded plugin with a specified name from OpenVINO Runtime
* The method is needed to remove plugin instance and free its resources. If plugin for a
* @brief Unloads the previously loaded plugin identified by @p device_name from OpenVINO Runtime
* The method is needed to remove loaded plugin instance and free its resources. If plugin for a
* specified device has not been created before, the method throws an exception.
*
* @param device_name Device name identifying plugin to remove from OpenVINO Runtime
* @note This method does not remove plugin from the plugins known to OpenVINO Core object.
* @param device_name A device name identifying plugin to remove from OpenVINO Runtime
*/
void unload_plugin(const std::string& device_name);
/** @brief Registers plugin to OpenVINO Runtime Core instance using XML configuration file with
/** @brief Registers a device plugin to OpenVINO Runtime Core instance using XML configuration file with
* plugins description.
*
* XML file has the following structure:
@ -370,29 +389,30 @@ public:
* </ie>
* ```
*
* - `name` identifies name of device enabled by plugin
* - `location` specifies absolute path to dynamic library with plugin. A path can also be relative to inference
* engine shared library. It allows to have common config for different systems with different configurations.
* - Properties are set to plugin via the `set_config` method.
* - Extensions are set to plugin via the `add_extension` method.
* - `name` identifies name of device enabled by a plugin
* - `location` specifies absolute path to dynamic library with a plugin.
* A path can also be relative to inference engine shared library. It allows to have common config
* for different systems with different configurations.
* - `properties` are set to a plugin via the ov::runtime::Core::set_config method.
* - `extensions` are set to a plugin via the ov::runtime::Core::add_extension method.
*
* @param xml_config_file A path to .xml file with plugins to register.
*/
void register_plugins(const std::string& xml_config_file);
/**
* @brief Create a new shared context object on specified accelerator device
* using specified plugin-specific low level device API parameters (device handle, pointer, etc.)
* @param device_name Name of a device to create new shared context on.
* @param params Map of device-specific shared context parameters.
* @return A shared pointer to a created remote context.
* @brief Create a new remote shared context object on specified accelerator device
* using specified plugin-specific low level device API parameters (device handle, pointer, context, etc.)
* @param device_name A name of a device to create a new shared context on.
* @param params A map of device-specific shared context parameters.
* @return A reference to a created remote context.
*/
RemoteContext create_context(const std::string& device_name, const ParamMap& params);
/**
* @brief Get a pointer to default(plugin-supplied) shared context object for specified accelerator device.
* @param device_name - A name of a device to get create shared context from.
* @return A shared pointer to a default remote context.
* @brief Get a pointer to default (plugin-supplied) shared context object for specified accelerator device.
* @param device_name A name of a device to get a default shared context from.
* @return A reference to a default remote context.
*/
RemoteContext get_default_context(const std::string& device_name);
};

9
src/inference/include/openvino/runtime/exception.hpp
View File

@ -9,12 +9,17 @@
namespace ov {
namespace runtime {
/// Thrown in case of canceled asynchronous operation
/**
* @brief Thrown in case of cancel;ed asynchronous operation
*/
class OPENVINO_RUNTIME_API Cancelled : public Exception {
using Exception::Exception;
};
/// Thrown in case of busy infer request
/**
* @brief Thrown in case of calling InferRequest methods while the request is busy with compute operation.
*/
class OPENVINO_RUNTIME_API Busy : public Exception {
using Exception::Exception;
};

183
src/inference/include/openvino/runtime/infer_request.hpp
View File

@ -3,7 +3,7 @@
//
/**
* @brief A header file that provides wrapper classes for infer requests and callbacks.
* @brief A header file that provides InferRequest.
*
* @file openvino/runtime/infer_request.hpp
*/
@ -29,9 +29,7 @@ namespace runtime {
class CompiledModel;
/**
* @brief This is an interface of asynchronous infer request
*
* It can throw exceptions safely for the application, where it is properly handled.
* @brief This is a class of infer request which can be run in asynchronous or synchronous manners.
*/
class OPENVINO_RUNTIME_API InferRequest {
std::shared_ptr<InferenceEngine::IInferRequestInternal> _impl;
@ -47,158 +45,183 @@ class OPENVINO_RUNTIME_API InferRequest {
friend class ov::runtime::CompiledModel;
public:
/// @brief Default constructor
/**
* @brief Default constructor
*/
InferRequest() = default;
/// @brief Default copy constructor
/// @param other other InferRequest object
/**
* @brief Default copy constructor
* @param other other InferRequest object
*/
InferRequest(const InferRequest& other) = default;
/// @brief Default copy assignment operator
/// @param other other InferRequest object
/// @return reference to the current object
/**
* @brief Default copy assignment operator
* @param other Another InferRequest object
* @return A reference to the current object
*/
InferRequest& operator=(const InferRequest& other) = default;
/// @brief Default move constructor
/// @param other other InferRequest object
/**
* @brief Default move constructor
* @param other other InferRequest object
*/
InferRequest(InferRequest&& other) = default;
/// @brief Default move assignment operator
/// @param other other InferRequest object
/// @return reference to the current object
/**
* @brief Default move assignment operator
* @param other other InferRequest object
* @return reference to the current object
*/
InferRequest& operator=(InferRequest&& other) = default;
/**
* @brief Destructor presereves unload order of implementation object and reference to library
* @brief Destructor preserves unloading order of implementation object and reference to library
* @note To preserve destruction order inside default generated assignment operator we store `_impl` before `_so`.
* And use destructor to remove implementation object before reference to library explicitly
*/
~InferRequest();
/**
* @brief Sets input/output data to infer
* @brief Sets input/output tensor to infer on
*
* @param name Name of input or output tensor.
* @param tensor Reference to input or output tensor. The type of a tensor must match the model input/output
* precision and size.
* @param tensor_name Name of input or output tensor.
* @param tensor Reference to a tensor. The element_type and shape of a tensor must match
* the model's input/output element_type and size.
*/
void set_tensor(const std::string& tensor_name, const Tensor& tensor);
void set_tensor(const std::string& name, const Tensor& tensor);
/**
* @brief Sets input/output data to infer
*
* @param port Port of input or output tensor.
* @param tensor Reference to input or output tensor. The type of a tensor must match the model input/output
* precision and size.
* @brief Sets input/output tensor to infer
* @param port Port of input or output tensor. Note, that the ports get from the following methods can be used:
* - ov::Model::input()
* - ov::Model::inputs()
* - ov::Model::outputs()
* - ov::Model::outputs()
* - ov::runtime::CompiledModel::input()
* - ov::runtime::CompiledModel::inputs()
* - ov::runtime::CompiledModel::outputs()
* - ov::runtime::CompiledModel::outputs()
* @param tensor Reference to a tensor. The element_type and shape of a tensor must match
* the model's input/output element_type and size.
*/
void set_tensor(const ov::Output<const ov::Node>& port, const Tensor& tensor);
/**
* @brief Sets input/output data to infer
*
* @param port Port of input or output tensor.
* @param tensor Reference to input or output tensor. The type of a tensor must match the model input/output
* precision and size.
* @brief Sets input/output tensor to infer
* @param port Port of input or output tensor. Note, that the ports get from the following methods can be used:
* - ov::Model::input()
* - ov::Model::inputs()
* - ov::Model::outputs()
* - ov::Model::outputs()
* - ov::runtime::CompiledModel::input()
* - ov::runtime::CompiledModel::inputs()
* - ov::runtime::CompiledModel::outputs()
* - ov::runtime::CompiledModel::outputs()
* @param tensor Reference to a tensor. The element_type and shape of a tensor must match
* the model's input/output element_type and size.
*/
void set_tensor(const ov::Output<ov::Node>& port, const Tensor& tensor);
/**
* @brief Sets batch of tensors for input data to infer by input name
* Model input shall have batch dimension and number of 'tensors' shall match with batch size
* Current version supports set tensors to model inputs only. In case if `name` is associated
* with output (or any other non-input node) - exception will be thrown
* @brief Sets batch of tensors for input data to infer by tensor name
* Model input shall have batch dimension and number of @p tensors shall match with batch size
* Current version supports set tensors to model inputs only. In case if @p tensor_name is associated
* with output (or any other non-input node) - an exception will be thrown
*
* @param name Name of input tensor.
* @param tensor_name Name of input tensor.
* @param tensors Input tensors for batched infer request. The type of each tensor must match the model
* input precision and size (except batch dimension). Total size of tensors shall match with input's size
* input element type and shape (except batch dimension). Total size of tensors shall match with input's size
*/
void set_tensors(const std::string& name, const std::vector<Tensor>& tensors);
void set_tensors(const std::string& tensor_name, const std::vector<Tensor>& tensors);
/**
* @brief Sets batch of tensors for input data to infer by input name
* Model input shall have batch dimension and number of 'tensors' shall match with batch size
* Current version supports set tensors to model inputs only. In case if `name` is associated
* with output (or any other non-input node) - exception will be thrown
* @brief Sets batch of tensors for input data to infer by input port
* Model input shall have batch dimension and number of @p tensors shall match with batch size
* Current version supports set tensors to model inputs only. In case if @p port is associated
* with output (or any other non-input node) - an exception will be thrown
*
* @param port Port of input tensor.
* @param tensors Input tensors for batched infer request. The type of each tensor must match the model
* input precision and size (except batch dimension). Total size of tensors shall match with input's size
* input element type and shape (except batch dimension). Total size of tensors shall match with input's size
*/
void set_tensors(const ov::Output<const ov::Node>& port, const std::vector<Tensor>& tensors);
/**
* @brief Sets input tensor to infer
*
* @param idx Index of input tensor.
* @param tensor Reference to input tensor. The type of a tensor must match the model input precision and size.
* @param idx Index of input tensor. If @p idx is greater than number of model inputs, an exception is thrown
* @param tensor Reference to a tensor. The element_type and shape of a tensor must match
* the model's input/output element_type and size.
*/
void set_input_tensor(size_t idx, const Tensor& tensor);
/**
* @brief Sets input tensor to infer models with single input
*
* @param tensor Reference to input tensor. If model has several inputs, an exception is thrown.
* @note If model has several inputs, an exception is thrown
* @param tensor Reference to input tensor.
*/
void set_input_tensor(const Tensor& tensor);
/**
* @brief Sets batch of tensors for single input data
* Model input shall have batch dimension and number of 'tensors' shall match with batch size
* Model input shall have batch dimension and number of @p tensors shall match with batch size
*
* @param tensors Input tensors for batched infer request. The type of each tensor must match the model
* input precision and size (except batch dimension). Total size of tensors shall match with input's size
* input element type and shape (except batch dimension). Total size of tensors shall match with input's size
*/
void set_input_tensors(const std::vector<Tensor>& tensors);
/**
* @brief Sets batch of tensors for input data to infer by input name
* Model input shall have batch dimension and number of 'tensors' shall match with batch size
* Model input shall have batch dimension and number of @p tensors shall match with batch size
*
* @param idx Name of input tensor.
* @param tensors Input tensors for batched infer request. The type of each tensor must match the model
* input precision and size (except batch dimension). Total size of tensors shall match with input's size
* input element type and shape (except batch dimension). Total size of tensors shall match with input's size
*/
void set_input_tensors(size_t idx, const std::vector<Tensor>& tensors);
/**
* @brief Sets output tensor to infer
*
* @note An index of input preserved accross ov::Model, ov::runtime::CompiledModel and ov::runtime::InferRequest
* @param idx Index of output tensor.
* @param tensor Reference to output tensor. The type of a tensor must match the model output precision and size.
* @param tensor Reference to output tensor. The type of a tensor must match the model output element type and
* shape.
*/
void set_output_tensor(size_t idx, const Tensor& tensor);
/**
* @brief Sets output tensor to infer models with single output
*
* @param tensor Reference to output tensor. If model has several outputs, an exception is thrown.
* @note If model has several outputs, an exception is thrown.
* @param tensor Reference to output tensor.
*/
void set_output_tensor(const Tensor& tensor);
/**
* @brief Gets input/output tensor for inference
*
* @param name A name of tensor to get
* @return A Tensor with a name @p name. If a tensor is not found, an exception is thrown.
* @brief Gets input/output tensor for inference by tensor name
* @param tensor_name A name of tensor to get
* @return A Tensor with a name @p tensor_name. If a tensor is not found, an exception is thrown.
*/
Tensor get_tensor(const std::string& tensor_name);
Tensor get_tensor(const std::string& name);
/**
* @brief Gets input/output tensor for inference
*
* @note If a tensor with specified @p port is not found, an exception is thrown
* @param port Port of tensor to get
* @return A Tensor for the port @p port. If a tensor with specified @p port is not found, an exception is thrown.
* @return A Tensor for the port @p port.
*/
Tensor get_tensor(const ov::Output<const ov::Node>& port);
/**
* @brief Gets input/output tensor for inference
*
* @note If a tensor with specified @p port is not found, an exception is thrown
* @param port Port of tensor to get
* @return A Tensor for the port @p port. If a tensor with specified @p port is not found, an exception is thrown.
* @return A Tensor for the port @p port.
*/
Tensor get_tensor(const ov::Output<ov::Node>& port);
/**
* @brief Gets input tensor for inference
*
@ -206,8 +229,8 @@ public:
* @return A Tensor with an input index @p idx. If a tensor with specified @p idx is not found, an exception is
* thrown.
*/
Tensor get_input_tensor(size_t idx);
/**
* @brief Gets input tensor for inference
*
@ -233,9 +256,8 @@ public:
/**
* @brief Infers specified input(s) in synchronous mode
*
* @note blocks all methods of InferRequest while request is ongoing (running or waiting in queue)
*
* Calling any method will lead to throwning ov::runtime::Busy exception
*/
void infer();
@ -245,17 +267,16 @@ public:
void cancel();
/**
* @brief Queries performance measures per layer to get feedback of what is the most time consuming layer
*
* @brief Queries performance measures per layer to get feedback of what is the most time consuming operation
* @note not all plugins provide meaningful data
* @return Vector of profiling information for operations in model
*/
std::vector<ProfilingInfo> get_profiling_info() const;
/**
* @brief Start inference of specified input(s) in asynchronous mode
*
* @brief Starts inference of specified input(s) in asynchronous mode
* @note It returns immediately. Inference starts also immediately.
* Calling any method while the request in a running state will lead to throwning ov::runtime::Busy exception
*/
void start_async();
@ -275,8 +296,7 @@ public:
bool wait_for(const std::chrono::milliseconds timeout);
/**
* @brief Sets a callback function that will be called on success or failure of asynchronous request
*
* @brief Sets a callback std::function that will be called on success or failure of asynchronous request
* @param callback callback object which will be called on when inference finish.
*/
void set_callback(std::function<void(std::exception_ptr)> callback);
@ -284,14 +304,13 @@ public:
/**
* @brief Gets state control interface for given infer request.
*
* State control essential for recurrent modells
* @return A vector of Memory State objects
* State control essential for recurrent models
* @return A vector of Variable State objects
*/
std::vector<VariableState> query_state();
/**
* @brief Returns compiled model that creates this inference request
*
* @return Compiled model object
*/
CompiledModel get_compiled_model();
@ -310,15 +329,17 @@ public:
/**
* @brief Compares whether this request wraps the same impl underneath
* @param other Another inference request
* @return true if current InferRequest object doesn't wrap the same impl as the operator's arg
*/
bool operator!=(const InferRequest&) const noexcept;
bool operator!=(const InferRequest& other) const noexcept;
/**
* @brief Compares whether this request wraps the same impl underneath
* @param other Another inference request
* @return true if current InferRequest object wraps the same impl as the operator's arg
*/
bool operator==(const InferRequest&) const noexcept;
bool operator==(const InferRequest& other) const noexcept;
};
} // namespace runtime
} // namespace ov

10
src/inference/include/openvino/runtime/intel_gpu/ocl/dx.hpp
View File

@ -40,7 +40,7 @@ namespace ocl {
class D3DBufferTensor : public ClBufferTensor {
public:
/**
* @brief Checks that type defined runtime paramters are presented in remote object
* @brief Checks that type defined runtime parameters are presented in remote object
* @param tensor a tensor to check
*/
static void type_check(const Tensor& tensor) {
@ -67,7 +67,7 @@ public:
class D3DSurface2DTensor : public ClImage2DTensor {
public:
/**
* @brief Checks that type defined runtime paramters are presented in remote object
* @brief Checks that type defined runtime parameters are presented in remote object
* @param remote_tensor remote tensor to check
*/
static void type_check(const Tensor& remote_tensor) {
@ -98,7 +98,7 @@ public:
* @brief This class represents an abstraction for GPU plugin remote context
* which is shared with Direct3D 11 device.
* The plugin object derived from this class can be obtained either with
* ExecutableNetwork::get_context() or Core::create_context() calls.
* CompiledModel::get_context() or Core::create_context() calls.
* @note User can also obtain OpenCL context handle from this class.
*/
class D3DContext : public ClContext {
@ -107,8 +107,8 @@ public:
using ClContext::create_tensor;
/**
* @brief Checks that type defined runtime paramters are presented in remote object
* @param remote_context remote context to check
* @brief Checks that type defined runtime parameters are presented in remote object
* @param remote_context A remote context to check
*/
static void type_check(const RemoteContext& remote_context) {
RemoteContext::type_check(

12
src/inference/include/openvino/runtime/intel_gpu/ocl/ocl.hpp
View File

@ -38,7 +38,7 @@ using gpu_handle_param = void*;
class ClBufferTensor : public RemoteTensor {
public:
/**
* @brief Checks that type defined runtime paramters are presented in remote object
* @brief Checks that type defined runtime parameters are presented in remote object
* @param tensor a tensor to check
*/
static void type_check(const Tensor& tensor) {
@ -82,7 +82,7 @@ public:
class ClImage2DTensor : public RemoteTensor {
public:
/**
* @brief Checks that type defined runtime paramters are presented in remote object
* @brief Checks that type defined runtime parameters are presented in remote object
* @param tensor a tensor to check
*/
static void type_check(const Tensor& tensor) {
@ -126,7 +126,7 @@ public:
class USMTensor : public RemoteTensor {
public:
/**
* @brief Checks that type defined runtime paramters are presented in remote object
* @brief Checks that type defined runtime parameters are presented in remote object
* @param tensor a tensor to check
*/
static void type_check(const Tensor& tensor) {
@ -151,7 +151,7 @@ public:
* @brief This class represents an abstraction for GPU plugin remote context
* which is shared with OpenCL context object.
* The plugin object derived from this class can be obtained either with
* ExecutableNetwork::get_context() or Core::create_context() calls.
* CompiledModel::get_context() or Core::create_context() calls.
*/
class ClContext : public RemoteContext {
protected:
@ -164,8 +164,8 @@ public:
// Needed to make create_tensor overloads from base class visible for user
using RemoteContext::create_tensor;
/**
* @brief Checks that type defined runtime paramters are presented in remote object
* @param remote_context remote context to check
* @brief Checks that type defined runtime parameters are presented in remote object
* @param remote_context A remote context to check
*/
static void type_check(const RemoteContext& remote_context) {
RemoteContext::type_check(remote_context,

2
src/inference/include/openvino/runtime/intel_gpu/ocl/ocl_wrapper.hpp
View File

@ -3,7 +3,7 @@
//
/**
* @brief a header that defines wrappers for internal GPU plugin-specific
* @brief A header that defines wrappers for internal GPU plugin-specific
* OpenCL context and OpenCL shared memory tensors
*
* @file ocl_wrapper.hpp

8
src/inference/include/openvino/runtime/intel_gpu/ocl/va.hpp
View File

@ -38,7 +38,7 @@ namespace ocl {
class VASurfaceTensor : public ClImage2DTensor {
public:
/**
* @brief Checks that type defined runtime paramters are presented in remote object
* @brief Checks that type defined runtime parameters are presented in remote object
* @param tensor a tensor to check
*/
static void type_check(const Tensor& tensor) {
@ -68,7 +68,7 @@ public:
* @brief This class represents an abstraction for GPU plugin remote context
* which is shared with VA display object.
* The plugin object derived from this class can be obtained either with
* ExecutableNetwork::get_context() or Core::create_context() calls.
* CompiledModel::get_context() or Core::create_context() calls.
* @note User can also obtain OpenCL context handle from this class.
*/
class VAContext : public ClContext {
@ -77,8 +77,8 @@ public:
using ClContext::create_tensor;
/**
* @brief Checks that type defined runtime paramters are presented in remote object
* @param remote_context remote context to check
* @brief Checks that type defined runtime parameters are presented in remote object
* @param remote_context A remote context to check
*/
static void type_check(const RemoteContext& remote_context) {
RemoteContext::type_check(

4
src/inference/include/openvino/runtime/profiling_info.hpp
View File

@ -17,9 +17,9 @@ namespace ov {
namespace runtime {
/**
* @struct ProfilingInfo
* @brief Represents basic inference profiling information per node.
* @brief Represents basic inference profiling information per operation.
*
* If the node is executed using tiling, the sum time per each tile is indicated as the total execution time.
* If the operation is executed using tiling, the sum time per each tile is indicated as the total execution time.
* Due to parallel execution, the total execution time for all nodes might be greater than the total inference time.
*/
struct ProfilingInfo {

51
src/inference/include/openvino/runtime/remote_context.hpp
View File

@ -3,8 +3,7 @@
//
/**
* @brief This is a header file for the OpenVINO Runtime RemoteContext class
*
* @brief A header file for the OpenVINO Runtime RemoteContext class
* @file openvino/runtime/remote_context.hpp
*/
#pragma once
@ -31,9 +30,9 @@ class CompiledModel;
/**
* @brief This class represents an abstraction
* for remote (non-CPU) accelerator device-specific execution context.
* Such context represents a scope on the device within which executable
* networks and remote memory blobs can exist, function and exchange data.
* for remote (non-CPU) accelerator device-specific inference context.
* Such context represents a scope on the device within which compiled
* models and remote memory tensors can exist, function and exchange data.
*/
class OPENVINO_RUNTIME_API RemoteContext {
protected:
@ -51,37 +50,47 @@ protected:
friend class ov::runtime::CompiledModel;
public:
/// @brief Default constructor
/**
* @brief Default constructor
*/
RemoteContext() = default;
/// @brief Default copy constructor
/// @param other other RemoteContext object
/**
* @brief Default copy constructor
* @param other other RemoteContext object
*/
RemoteContext(const RemoteContext& other) = default;
/// @brief Default copy assignment operator
/// @param other other RemoteContext object
/// @return reference to the current object
/**
* @brief Default copy assignment operator
* @param other other RemoteContext object
* @return reference to the current object
*/
RemoteContext& operator=(const RemoteContext& other) = default;
/// @brief Default move constructor
/// @param other other RemoteContext object
/**
* @brief Default move constructor
* @param other other RemoteContext object
*/
RemoteContext(RemoteContext&& other) = default;
/// @brief Default move assignment operator
/// @param other other RemoteContext object
/// @return reference to current object
/**
* @brief Default move assignment operator
* @param other other RemoteContext object
* @return reference to current object
*/
RemoteContext& operator=(RemoteContext&& other) = default;
/**
* @brief Destructor presereves unload order of implementation object and reference to library
* @brief Destructor preserves unloading order of implementation object and reference to library
*/
~RemoteContext();
/**
* @brief Checks openvino remote type
* @brief Internal method: Checks remote type
* @param remote_context a remote context which type will be checked
* @param type_info map with remote object runtime info
* @throw Exception if type check with specified paramters is not pass
* @throw Exception if type check with specified parameters failed
*/
static void type_check(const RemoteContext& remote_context,
const std::map<std::string, std::vector<std::string>>& type_info = {});
@ -121,7 +130,7 @@ public:
/**
* @brief Returns name of the device on which underlying object is allocated.
* Abstract method.
* @return A device name string in fully specified format `<device_name>[.<device_id>[.<tile_id>]]`.
* @return A device name string in fully specified format `<device_name>[.<device_id>[.<tile_id>]]` (e.g. GPU.0.1).
*/
std::string get_device_name() const;
@ -148,7 +157,7 @@ public:
ParamMap get_params() const;
/**
* @brief This function is used to create host tensor object friendly for the device in current context
* @brief This method is used to create host tensor object friendly for the device in current context
* For example, GPU context may allocate USM host memory (if corresponding extension is available)
* which could be more efficient than regular host memory.
* @param type Tensor element type

10
src/inference/include/openvino/runtime/remote_tensor.hpp
View File

@ -19,8 +19,6 @@ class RemoteContext;
/**
* @brief Remote memory access and interpretation API
*
* It can throw exceptions safely for the application, where it is properly handled.
*/
class OPENVINO_RUNTIME_API RemoteTensor : public Tensor {
using Tensor::Tensor;
@ -31,10 +29,16 @@ public:
* @brief Checks openvino remote type
* @param tensor tensor which type will be checked
* @param type_info map with remote object runtime info
* @throw Exception if type check with specified paramters is not pass
* @throw Exception if type check with specified parameters failed
*/
static void type_check(const Tensor& tensor, const std::map<std::string, std::vector<std::string>>& type_info = {});
/**
* @brief Access of host memory is not available for RemoteTensor
* To access a device specific memory, cast to specific RemoteTensor derived object and works with its
* properties or parse device memory properies via RemoteTensor::get_params
* @return Nothing, throws an exception
*/
void* data(const element::Type) = delete;
template <typename T>

7
src/inference/include/openvino/runtime/variable_state.hpp
View File

@ -3,8 +3,7 @@
//
/**
* @brief A header file that provides VariableState
*
* @brief A header file that provides ov::runtime::VariableState
* @file openvino/runtime/variable_state.hpp
*/
@ -50,7 +49,7 @@ public:
VariableState() = default;
/**
* @brief Destructor presereves unload order of implementation object and reference to library
* @brief Destructor preserves unloading order of implementation object and reference to library
*/
~VariableState();
@ -69,7 +68,7 @@ public:
/**
* @brief Returns the value of the variable state.
* @return A blob representing a state
* @return A tensor representing a state
*/
Tensor get_state() const;