Added openvino infer request API (#7151)

inference-engine/src/inference_engine/include/openvino/runtime/common.hpp

@@ -3,12 +3,13 @@
 //
 
 /**
- * @brief This is a header file for the OpenVINO Runtime common aliases that depend only from external API
+ * @brief This is a header file for the OpenVINO Runtime common aliases and data types
  *
  * @file openvino/runtime/common.hpp
  */
 #pragma once
 
+#include <chrono>
 #include <map>
 #include <string>
 

inference-engine/src/inference_engine/include/openvino/runtime/infer_request.hpp

@@ -0,0 +1,178 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+/**
+ * @brief A header file that provides wrapper classes for infer requests and callbacks.
+ *
+ * @file infer_request.hpp
+ */
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+
+#include "common.hpp"
+#include "profiling_info.hpp"
+#include "variable_state.hpp"
+
+namespace InferenceEngine {
+class IInferRequestInternal;
+class Blob;
+}  // namespace InferenceEngine
+
+namespace ov {
+namespace runtime {
+/**
+ * @brief This is an interface of asynchronous infer request
+ *
+ * It can throw exceptions safely for the application, where it is properly handled.
+ */
+class INFERENCE_ENGINE_API_CLASS(InferRequest) {
+    std::shared_ptr<SharedObject> _so;
+    std::shared_ptr<ie::IInferRequestInternal> _impl;
+
+    /**
+     * @brief Constructs InferRequest from the initialized std::shared_ptr
+     * @param so Plugin to use. This is required to ensure that InferRequest can work properly even if plugin object is
+     * destroyed.
+     * @param impl Initialized shared pointer
+     */
+    InferRequest(const std::shared_ptr<SharedObject>& so, const std::shared_ptr<ie::IInferRequestInternal>& impl);
+    friend class ExecutableNetwork;
+
+public:
+    /**
+     * @brief Default constructor
+     */
+    InferRequest() = default;
+
+    /**
+     * @brief Sets input/output data to infer
+     *
+     * @note Memory allocation does not happen
+     * @param name Name of input or output blob.
+     * @param data Reference to input or output blob. The type of a blob must match the network input precision and
+     * size.
+     */
+    void set_blob(const std::string& name, const std::shared_ptr<ie::Blob>& data);
+
+    /**
+     * @brief Gets input/output data for inference
+     *
+     * @note Memory allocation does not happen
+     * @param name A name of Blob to get
+     * @return A shared pointer to a Blob with a name @p name. If a blob is not found, an exception is thrown.
+     */
+    std::shared_ptr<ie::Blob> get_blob(const std::string& name);
+
+    /**
+     * @brief Infers specified input(s) in synchronous mode
+     *
+     * @note blocks all methods of InferRequest while request is ongoing (running or waiting in queue)
+     *
+     */
+    void infer();
+
+    /**
+     * @brief Cancels inference request
+     */
+    void cancel();
+
+    /**
+     * @brief Queries performance measures per layer to get feedback of what is the most time consuming layer
+     *
+     * @note not all plugins provide meaningful data
+     * @return Vector of profiling information for layers in network
+     */
+    std::vector<ProfilingInfo> get_profiling_info() const;
+
+    /**
+     * @brief Sets input data to infer
+     *
+     * @note Memory allocation doesn't happen
+     * @param inputs A reference to a map of input blobs accessed by input names.
+     *        The type of Blob must correspond to the network input precision and size.
+     */
+    void set_input(const std::map<std::string, std::shared_ptr<ie::Blob>>& inputs);
+
+    /**
+     * @brief Sets data that will contain result of the inference
+     *
+     * @note Memory allocation doesn't happen
+     * @param results - a reference to a map of result blobs accessed by output names.
+     *        The type of Blob must correspond to the network output precision and size.
+     */
+    void set_output(const std::map<std::string, std::shared_ptr<ie::Blob>>& results);
+
+    /**
+     * @brief Sets new batch size when dynamic batching is enabled in executable network that created this request.
+     *
+     * @param batch new batch size to be used by all the following inference calls for this request.
+     */
+    void set_batch(const int batch);
+
+    /**
+     * @brief Start inference of specified input(s) in asynchronous mode
+     *
+     * @note It returns immediately. Inference starts also immediately.
+     */
+    void start_async();
+
+    /**
+     * @brief Waits for the result to become available. Blocks until the result
+     * becomes available
+     */
+    void wait();
+
+    /**
+     * @brief Waits for the result to become available. Blocks until specified timeout has elapsed or the result
+     * becomes available, whichever comes first.
+     *
+     * @param timeout Maximum duration in milliseconds to block for
+     * @return true if inference request is ready and false otherwise
+     */
+    bool wait_for(const std::chrono::milliseconds timeout);
+
+    /**
+     * @brief Sets a callback function that will be called on success or failure of asynchronous request
+     *
+     * @param callback callback object which will be called on when inference finish.
+     */
+    void set_callback(std::function<void(std::exception_ptr)> callback);
+
+    /**
+     * @brief Gets state control interface for given infer request.
+     *
+     * State control essential for recurrent networks
+     * @return A vector of Memory State objects
+     */
+    std::vector<VariableState> query_state();
+
+    /**
+     * @brief Checks if current InferRequest object is not initialized
+     * @return true if current InferRequest object is not initialized, false - otherwise
+     */
+    bool operator!() const noexcept;
+
+    /**
+     * @brief Checks if current InferRequest object is initialized
+     * @return true if current InferRequest object is initialized, false - otherwise
+     */
+    explicit operator bool() const noexcept;
+
+    /**
+     * @brief Compares whether this request wraps the same impl underneath
+     * @return true if current InferRequest object doesn't wrap the same impl as the operator's arg
+     */
+    bool operator!=(const InferRequest&) const noexcept;
+
+    /**
+     * @brief Compares whether this request wraps the same impl underneath
+     * @return true if current InferRequest object wraps the same impl as the operator's arg
+     */
+    bool operator==(const InferRequest&) const noexcept;
+};
+}  // namespace runtime
+}  // namespace ov

inference-engine/src/inference_engine/include/openvino/runtime/profiling_info.hpp

@@ -0,0 +1,65 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+/**
+ * @brief This is a header file for the ProfilingInfo objects that contains performance
+ *        metric for single node
+ *
+ * @file openvino/runtime/profiling_info.hpp
+ */
+#pragma once
+
+#include <chrono>
+#include <string>
+
+namespace ov {
+namespace runtime {
+/**
+ * @struct ProfilingInfo
+ * @brief Represents basic inference profiling information per node.
+ *
+ * If the node is executed using tiling, the sum time per each tile is indicated as the total execution time.
+ * Due to parallel execution, the total execution time for all nodes might be greater than the total inference time.
+ */
+struct ProfilingInfo {
+    /**
+     * @brief Defines the general status of the node
+     */
+    enum class Status {
+        NOT_RUN,        //!< A node is not executed
+        OPTIMIZED_OUT,  //!< A node is optimized out during graph optimization phase
+        EXECUTED        //!< A node is executed
+    };
+
+    /**
+     * @brief Defines a node status
+     */
+    Status status;
+
+    /**
+     * @brief The absolute time in microseconds that the node ran (in total)
+     */
+    std::chrono::microseconds real_time;
+    /**
+     * @brief The net host cpu time that the node ran
+     */
+    std::chrono::microseconds cpu_time;
+
+    /**
+     * @brief A name of node
+     */
+    std::string node_name;
+
+    /**
+     * @brief An execution type of unit
+     */
+    std::string exec_type;
+
+    /**
+     * @brief A node type
+     */
+    std::string node_type;
+};
+}  // namespace runtime
+}  // namespace ov

inference-engine/src/inference_engine/include/openvino/runtime/variable_state.hpp

@@ -0,0 +1,81 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+/**
+ * @brief A header file that provides VariableState
+ *
+ * @file variable_state.hpp
+ */
+
+#pragma once
+
+#include <ie_api.h>
+#include <ie_blob.h>
+
+#include <memory>
+#include <string>
+
+#include "common.hpp"
+
+namespace InferenceEngine {
+class IVariableStateInternal;
+class Blob;
+}  // namespace InferenceEngine
+
+namespace ov {
+namespace runtime {
+
+class SharedObject;
+class InferRequest;
+
+/**
+ * @brief VariableState class
+ */
+class INFERENCE_ENGINE_API_CLASS(VariableState) {
+    std::shared_ptr<SharedObject> _so;
+    std::shared_ptr<ie::IVariableStateInternal> _impl;
+
+    /**
+     * @brief Constructs VariableState from the initialized std::shared_ptr
+     * @param impl Initialized shared pointer
+     * @param so Optional: Plugin to use. This is required to ensure that VariableState can work properly even if plugin
+     * object is destroyed.
+     */
+    VariableState(const std::shared_ptr<SharedObject>& so, const std::shared_ptr<ie::IVariableStateInternal>& impl);
+
+    friend class ov::runtime::InferRequest;
+
+public:
+    /**
+     * @brief Default constructor
+     */
+    VariableState() = default;
+
+    /**
+     * @brief Reset internal variable state for relevant infer request,
+     * to a value specified as default for according ReadValue node
+     */
+    void reset();
+
+    /**
+     * @brief Gets name of current variable state, if length of array is not enough name is truncated by len, null
+     * terminator is inserted as well. As variable state name `variable_id` from according `ReadValue` used.
+     * @return A string representing a state name
+     */
+    std::string get_name() const;
+
+    /**
+     * @brief Returns the value of the variable state.
+     * @return A blob representing a state
+     */
+    std::shared_ptr<const ie::Blob> get_state() const;
+
+    /**
+     * @brief Sets the new state for the next inference.
+     * @param state The current state to set
+     */
+    void set_state(const std::shared_ptr<ie::Blob>& state);
+};
+}  // namespace runtime
+}  // namespace ov

inference-engine/src/inference_engine/src/cpp/ie_infer_request.cpp

@@ -10,8 +10,10 @@
 
 #include "cpp/exception2status.hpp"
 #include "cpp_interfaces/interface/ie_iinfer_request_internal.hpp"
+#include "details/ie_so_loader.h"
 #include "ie_infer_async_request_base.hpp"
 #include "ie_remote_context.hpp"
+#include "openvino/runtime/infer_request.hpp"
 
 namespace InferenceEngine {
 
@@ -21,7 +23,7 @@ namespace InferenceEngine {
     try {                                                                 \
         __VA_ARGS__                                                       \
     } catch (...) {                                                       \
-        details::Rethrow();                                               \
+        ::InferenceEngine::details::Rethrow();                            \
     }
 
 InferRequest::InferRequest(const details::SharedObjectLoader& so, const IInferRequestInternal::Ptr& impl)
@@ -190,3 +192,131 @@ bool InferRequest::operator==(const InferRequest& r) const noexcept {
 }
 
 }  // namespace InferenceEngine
+
+namespace ov {
+namespace runtime {
+
+InferRequest::InferRequest(const std::shared_ptr<SharedObject>& so, const ie::IInferRequestInternal::Ptr& impl)
+    : _so{so},
+      _impl{impl} {
+    IE_ASSERT(_impl != nullptr);
+}
+
+void InferRequest::set_blob(const std::string& name, const ie::Blob::Ptr& data) {
+    INFER_REQ_CALL_STATEMENT(_impl->SetBlob(name, data);)
+}
+
+ie::Blob::Ptr InferRequest::get_blob(const std::string& name) {
+    ie::Blob::Ptr blobPtr;
+    INFER_REQ_CALL_STATEMENT(blobPtr = _impl->GetBlob(name);)
+    std::string error = "Internal error: blob with name `" + name + "` is not allocated!";
+    const bool remoteBlobPassed = blobPtr->is<ie::RemoteBlob>();
+    if (blobPtr == nullptr)
+        IE_THROW() << error;
+    if (!remoteBlobPassed && blobPtr->buffer() == nullptr)
+        IE_THROW() << error;
+    return blobPtr;
+}
+
+void InferRequest::infer() {
+    INFER_REQ_CALL_STATEMENT(_impl->Infer();)
+}
+
+void InferRequest::cancel() {
+    INFER_REQ_CALL_STATEMENT(_impl->Cancel();)
+}
+
+std::vector<ProfilingInfo> InferRequest::get_profiling_info() const {
+    INFER_REQ_CALL_STATEMENT({
+        auto ieInfos = _impl->GetPerformanceCounts();
+        std::vector<ProfilingInfo> infos;
+        infos.reserve(ieInfos.size());
+        while (!ieInfos.empty()) {
+            auto itIeInfo = std::min_element(
+                std::begin(ieInfos),
+                std::end(ieInfos),
+                [](const decltype(ieInfos)::value_type& lhs, const decltype(ieInfos)::value_type& rhs) {
+                    return lhs.second.execution_index < rhs.second.execution_index;
+                });
+            IE_ASSERT(itIeInfo != ieInfos.end());
+            auto& ieInfo = itIeInfo->second;
+            infos.push_back(ProfilingInfo{});
+            auto& info = infos.back();
+            switch (ieInfo.status) {
+            case ie::InferenceEngineProfileInfo::NOT_RUN:
+                info.status = ProfilingInfo::Status::NOT_RUN;
+                break;
+            case ie::InferenceEngineProfileInfo::OPTIMIZED_OUT:
+                info.status = ProfilingInfo::Status::OPTIMIZED_OUT;
+                break;
+            case ie::InferenceEngineProfileInfo::EXECUTED:
+                info.status = ProfilingInfo::Status::OPTIMIZED_OUT;
+                break;
+            }
+            info.real_time = std::chrono::microseconds{ieInfo.realTime_uSec};
+            info.cpu_time = std::chrono::microseconds{ieInfo.cpu_uSec};
+            info.node_name = itIeInfo->first;
+            info.exec_type = std::string{ieInfo.exec_type};
+            info.node_type = std::string{ieInfo.layer_type};
+            ieInfos.erase(itIeInfo);
+        }
+        return infos;
+    })
+}
+
+void InferRequest::set_input(const ie::BlobMap& inputs) {
+    INFER_REQ_CALL_STATEMENT(for (auto&& input : inputs) { _impl->SetBlob(input.first, input.second); })
+}
+
+void InferRequest::set_output(const ie::BlobMap& results) {
+    INFER_REQ_CALL_STATEMENT(for (auto&& result : results) { _impl->SetBlob(result.first, result.second); })
+}
+
+void InferRequest::set_batch(const int batch) {
+    INFER_REQ_CALL_STATEMENT(_impl->SetBatch(batch);)
+}
+
+void InferRequest::start_async() {
+    INFER_REQ_CALL_STATEMENT(_impl->StartAsync();)
+}
+
+void InferRequest::wait() {
+    INFER_REQ_CALL_STATEMENT(_impl->Wait(ie::InferRequest::RESULT_READY);)
+}
+
+bool InferRequest::wait_for(const std::chrono::milliseconds timeout) {
+    INFER_REQ_CALL_STATEMENT(return _impl->Wait(timeout.count()) == ie::OK;)
+}
+
+void InferRequest::set_callback(std::function<void(std::exception_ptr)> callback) {
+    INFER_REQ_CALL_STATEMENT(_impl->SetCallback(std::move(callback));)
+}
+
+std::vector<VariableState> InferRequest::query_state() {
+    std::vector<VariableState> variable_states;
+    INFER_REQ_CALL_STATEMENT({
+        for (auto&& state : _impl->QueryState()) {
+            variable_states.emplace_back(VariableState{_so, state});
+        }
+    })
+    return variable_states;
+}
+
+bool InferRequest::operator!() const noexcept {
+    return !_impl;
+}
+
+InferRequest::operator bool() const noexcept {
+    return (!!_impl);
+}
+
+bool InferRequest::operator!=(const InferRequest& r) const noexcept {
+    return !(r == *this);
+}
+
+bool InferRequest::operator==(const InferRequest& r) const noexcept {
+    return r._impl == _impl;
+}
+
+}  // namespace runtime
+}  // namespace ov

inference-engine/src/inference_engine/src/cpp/ie_variable_state.cpp

@@ -6,6 +6,7 @@
 #include "cpp_interfaces/interface/ie_ivariable_state_internal.hpp"
 #include "details/ie_so_loader.h"
 #include "exception2status.hpp"
+#include "openvino/runtime/variable_state.hpp"
 
 #define VARIABLE_CALL_STATEMENT(...)                                    \
     if (_impl == nullptr)                                               \
@@ -13,7 +14,7 @@
     try {                                                               \
         __VA_ARGS__;                                                    \
     } catch (...) {                                                     \
-        details::Rethrow();                                             \
+        ::InferenceEngine::details::Rethrow();                          \
     }
 
 namespace InferenceEngine {
@@ -44,3 +45,31 @@ void VariableState::SetState(Blob::Ptr state) {
 }
 
 }  // namespace InferenceEngine
+
+namespace ov {
+namespace runtime {
+
+VariableState::VariableState(const std::shared_ptr<SharedObject>& so, const ie::IVariableStateInternal::Ptr& impl)
+    : _so{so},
+      _impl{impl} {
+    IE_ASSERT(_impl != nullptr);
+}
+
+void VariableState::reset() {
+    VARIABLE_CALL_STATEMENT(_impl->Reset());
+}
+
+std::string VariableState::get_name() const {
+    VARIABLE_CALL_STATEMENT(return _impl->GetName());
+}
+
+ie::Blob::CPtr VariableState::get_state() const {
+    VARIABLE_CALL_STATEMENT(return _impl->GetState());
+}
+
+void VariableState::set_state(const ie::Blob::Ptr& state) {
+    VARIABLE_CALL_STATEMENT(_impl->SetState(state));
+}
+
+}  // namespace runtime
+}  // namespace ov

inference-engine/src/inference_engine/src/os/lin/lin_shared_object_loader.cpp

@@ -8,6 +8,7 @@
 
 #include "details/ie_so_loader.h"
 #include "file_utils.h"
+#include "shared_object.hpp"
 
 namespace InferenceEngine {
 namespace details {
@@ -71,3 +72,33 @@ void* SharedObjectLoader::get_symbol(const char* symbolName) const {
 
 }  // namespace details
 }  // namespace InferenceEngine
+
+namespace ov {
+namespace runtime {
+SharedObject::SharedObject(const char* path) {
+    shared_object = dlopen(path, RTLD_NOW);
+
+    if (shared_object == nullptr)
+        IE_THROW() << "Cannot load library '" << path << "': " << dlerror();
+}
+
+#ifdef ENABLE_UNICODE_PATH_SUPPORT
+SharedObject::SharedObject(const wchar_t* path) : SharedObject(FileUtils::wStringtoMBCSstringChar(path).c_str()) {}
+#endif  // ENABLE_UNICODE_PATH_SUPPORT
+
+SharedObject::~SharedObject() {
+    if (0 != dlclose(shared_object)) {
+        std::cerr << "dlclose failed: " << dlerror() << std::endl;
+    }
+}
+
+void* SharedObject::get_symbol(const char* symbolName) const {
+    void* procAddr = nullptr;
+
+    procAddr = dlsym(shared_object, symbolName);
+    if (procAddr == nullptr)
+        IE_THROW(NotFound) << "dlSym cannot locate method '" << symbolName << "': " << dlerror();
+    return procAddr;
+}
+}  // namespace runtime
+}  // namespace ov

inference-engine/src/inference_engine/src/os/win/win_shared_object_loader.cpp

@@ -5,6 +5,7 @@
 #include "ie_common.h"
 #include "details/ie_so_loader.h"
 #include "file_utils.h"
+#include "shared_object.hpp"
 
 //
 // LoadLibraryA, LoadLibraryW:
@@ -274,3 +275,112 @@ void* SharedObjectLoader::get_symbol(const char* symbolName) const {
 
 }  // namespace details
 }  // namespace InferenceEngine
+
+
+namespace ov {
+namespace runtime {
+SharedObject::SharedObject(const char* path) {
+    using GetDllDirectoryA_Fnc = DWORD(*)(DWORD, LPSTR);
+    GetDllDirectoryA_Fnc IEGetDllDirectoryA = nullptr;
+    if (HMODULE hm = GetModuleHandleW(L"kernel32.dll")) {
+        IEGetDllDirectoryA = reinterpret_cast<GetDllDirectoryA_Fnc>(GetProcAddress(hm, "GetDllDirectoryA"));
+    }
+#if !WINAPI_PARTITION_SYSTEM
+    // ExcludeCurrentDirectory
+    if (IEGetDllDirectoryA && IEGetDllDirectoryA(0, NULL) <= 1) {
+        SetDllDirectoryA("");
+    }
+    // LoadPluginFromDirectory
+    if (IEGetDllDirectoryA) {
+        DWORD nBufferLength = IEGetDllDirectoryA(0, NULL);
+        std::vector<CHAR> lpBuffer(nBufferLength);
+        IEGetDllDirectoryA(nBufferLength, &lpBuffer.front());
+
+        // GetDirname
+        auto dirname = [&] {
+            auto pos = strchr(path, '\\');
+            if (pos == nullptr) {
+                return std::string{path};
+            }
+            std::string original(path);
+            original[pos - path] = 0;
+            return original;
+        } ();
+
+        SetDllDirectoryA(dirname.c_str());
+        shared_object = LoadLibraryA(path);
+
+        SetDllDirectoryA(&lpBuffer.front());
+    }
+#endif
+    if (!shared_object) {
+        shared_object = LoadLibraryA(path);
+    }
+    if (!shared_object) {
+        char cwd[1024];
+        IE_THROW() << "Cannot load library '" << path << "': " << GetLastError()
+            << " from cwd: " << _getcwd(cwd, sizeof(cwd));
+    }
+}
+
+#ifdef ENABLE_UNICODE_PATH_SUPPORT
+SharedObject::SharedObject(const wchar_t* path) {
+    using GetDllDirectoryW_Fnc = DWORD(*)(DWORD, LPWSTR);
+    static GetDllDirectoryW_Fnc IEGetDllDirectoryW = nullptr;
+    if (HMODULE hm = GetModuleHandleW(L"kernel32.dll")) {
+        IEGetDllDirectoryW = reinterpret_cast<GetDllDirectoryW_Fnc>(GetProcAddress(hm, "GetDllDirectoryW"));
+    }
+    // ExcludeCurrentDirectory
+#if !WINAPI_PARTITION_SYSTEM
+    if (IEGetDllDirectoryW && IEGetDllDirectoryW(0, NULL) <= 1) {
+        SetDllDirectoryW(L"");
+    }
+    if (IEGetDllDirectoryW) {
+        DWORD nBufferLength = IEGetDllDirectoryW(0, NULL);
+        std::vector<WCHAR> lpBuffer(nBufferLength);
+        IEGetDllDirectoryW(nBufferLength, &lpBuffer.front());
+
+        auto dirname = [&] {
+            auto pos = wcsrchr(path, '\\');
+            if (pos == nullptr) {
+                return std::wstring{path};
+            }
+            std::wstring original(path);
+            original[pos - path] = 0;
+            return original;
+        } ();
+        SetDllDirectoryW(dirname.c_str());
+        shared_object = LoadLibraryW(path);
+
+        SetDllDirectoryW(&lpBuffer.front());
+    }
+#endif
+    if (!shared_object) {
+        shared_object = LoadLibraryW(path);
+    }
+    if (!shared_object) {
+        char cwd[1024];
+        IE_THROW() << "Cannot load library '" << FileUtils::wStringtoMBCSstringChar(std::wstring(path)) << "': " << GetLastError()
+                            << " from cwd: " << _getcwd(cwd, sizeof(cwd));
+    }
+}
+#endif
+
+SharedObject::~SharedObject() {
+    FreeLibrary(reinterpret_cast<HMODULE>(shared_object));
+}
+
+void* SharedObject::get_symbol(const char* symbolName) const {
+    if (!shared_object) {
+        IE_THROW() << "Cannot get '" << symbolName << "' content from unknown library!";
+    }
+    auto procAddr = reinterpret_cast<void*>(GetProcAddress(
+        reinterpret_cast<HMODULE>(const_cast<void*>(shared_object)), symbolName));
+    if (procAddr == nullptr)
+        IE_THROW(NotFound)
+            << "GetProcAddress cannot locate method '" << symbolName << "': " << GetLastError();
+
+    return procAddr;
+}
+}  // namespace runtime
+}  // namespace ov

inference-engine/src/plugin_api/shared_object.hpp

@@ -0,0 +1,44 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+/**
+ * @brief A header file for definition of abstraction over platform specific shared objects
+ * @file ie_system_conf.h
+ */
+
+#pragma once
+
+#include "ie_api.h"
+
+namespace ov {
+namespace runtime {
+struct INFERENCE_ENGINE_API_CLASS(SharedObject) {
+    void* shared_object = nullptr;
+
+    /**
+     * @brief Loads a library with the name specified.
+     * @param path Full or relative path to the plugin library
+     */
+    explicit SharedObject(const char* path);
+
+#ifdef ENABLE_UNICODE_PATH_SUPPORT
+    /**
+     * @brief Loads a library with the wide char name specified.
+     * @param path Full or relative path to the plugin library
+     */
+    explicit SharedObject(const wchar_t* path);
+#endif  // ENABLE_UNICODE_PATH_SUPPORT
+
+    ~SharedObject();
+
+    /**
+     * @brief Searches for a function symbol in the loaded module
+     * @param symbolName Name of the function to find
+     * @return A pointer to the function if found
+     * @throws Exception if the function is not found
+     */
+    void* get_symbol(const char* symbolName) const;
+};
+}  // namespace runtime
+}  // namespace ov

inference-engine/tests/functional/inference_engine/ov_infer_request_test.cpp

@@ -0,0 +1,75 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <gtest/gtest.h>
+
+#include <cpp/ie_infer_request.hpp>
+#include <openvino/runtime/infer_request.hpp>
+
+using namespace ::testing;
+using namespace std;
+using namespace InferenceEngine;
+using namespace InferenceEngine::details;
+
+
+TEST(InferRequestOVTests, throwsOnUninitializedSetBlob) {
+    ov::runtime::InferRequest req;
+    ASSERT_THROW(req.set_blob({}, {}), InferenceEngine::NotAllocated);
+}
+
+TEST(InferRequestOVTests, throwsOnUninitializedGetBlob) {
+    ov::runtime::InferRequest req;
+    ASSERT_THROW(req.get_blob({}), InferenceEngine::NotAllocated);
+}
+
+TEST(InferRequestOVTests, throwsOnUninitializedInfer) {
+    ov::runtime::InferRequest req;
+    ASSERT_THROW(req.infer(), InferenceEngine::NotAllocated);
+}
+
+TEST(InferRequestOVTests, throwsOnUninitializedGetPerformanceCounts) {
+    ov::runtime::InferRequest req;
+    ASSERT_THROW(req.get_profiling_info(), InferenceEngine::NotAllocated);
+}
+
+TEST(InferRequestOVTests, throwsOnUninitializedSetInput) {
+    ov::runtime::InferRequest req;
+    ASSERT_THROW(req.set_input({{}}), InferenceEngine::NotAllocated);
+}
+
+TEST(InferRequestOVTests, throwsOnUninitializedSetOutput) {
+    ov::runtime::InferRequest req;
+    ASSERT_THROW(req.set_output({{}}), InferenceEngine::NotAllocated);
+}
+
+TEST(InferRequestOVTests, throwsOnUninitializedSetBatch) {
+    ov::runtime::InferRequest req;
+    ASSERT_THROW(req.set_batch({}), InferenceEngine::NotAllocated);
+}
+
+TEST(InferRequestOVTests, throwsOnUninitializedStartAsync) {
+    ov::runtime::InferRequest req;
+    ASSERT_THROW(req.start_async(), InferenceEngine::NotAllocated);
+}
+
+TEST(InferRequestOVTests, throwsOnUninitializedWait) {
+    ov::runtime::InferRequest req;
+    ASSERT_THROW(req.wait(), InferenceEngine::NotAllocated);
+}
+
+TEST(InferRequestOVTests, throwsOnUninitializedWaitFor) {
+    ov::runtime::InferRequest req;
+    ASSERT_THROW(req.wait_for({}), InferenceEngine::NotAllocated);
+}
+
+TEST(InferRequestOVTests, throwsOnUninitializedSetCompletionCallback) {
+    ov::runtime::InferRequest req;
+    std::function<void(std::exception_ptr)> f;
+    ASSERT_THROW(req.set_callback(f), InferenceEngine::NotAllocated);
+}
+
+TEST(InferRequestOVTests, throwsOnUninitializedQueryState) {
+    ov::runtime::InferRequest req;
+    ASSERT_THROW(req.query_state(), InferenceEngine::NotAllocated);
+}

inference-engine/tests/functional/inference_engine/ov_shared_object_test.cpp

@@ -0,0 +1,61 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <gtest/gtest.h>
+
+#include <file_utils.h>
+#include "shared_object.hpp"
+#include <cpp/ie_plugin.hpp>
+
+using namespace ::testing;
+using namespace std;
+
+class SharedObjectOVTests: public ::testing::Test {
+protected:
+    std::string get_mock_engine_name() {
+        return FileUtils::makePluginLibraryName<char>(InferenceEngine::getIELibraryPath(),
+            std::string("mock_engine") + IE_BUILD_POSTFIX);
+    }
+
+    void loadDll(const string &libraryName) {
+        sharedObject.reset(new ov::runtime::SharedObject(libraryName.c_str()));
+    }
+    unique_ptr<ov::runtime::SharedObject> sharedObject;
+
+    using CreateF = void(std::shared_ptr<InferenceEngine::IInferencePlugin>&);
+
+    std::function<CreateF> make_std_function(const std::string& functionName) {
+        std::function<CreateF> ptr(reinterpret_cast<CreateF*>(sharedObject->get_symbol(functionName.c_str())));
+        return ptr;
+    }
+};
+
+TEST_F(SharedObjectOVTests, canLoadExistedPlugin) {
+    loadDll(get_mock_engine_name());
+    EXPECT_NE(nullptr, sharedObject.get());
+}
+
+TEST_F(SharedObjectOVTests, loaderThrowsIfNoPlugin) {
+    EXPECT_THROW(loadDll("wrong_name"), InferenceEngine::Exception);
+}
+
+TEST_F(SharedObjectOVTests, canFindExistedMethod) {
+    loadDll(get_mock_engine_name());
+
+    auto factory = make_std_function("CreatePluginEngine");
+    EXPECT_NE(nullptr, factory);
+}
+
+TEST_F(SharedObjectOVTests, throwIfMethodNofFoundInLibrary) {
+    loadDll(get_mock_engine_name());
+    EXPECT_THROW(make_std_function("wrong_function"), InferenceEngine::Exception);
+}
+
+TEST_F(SharedObjectOVTests, canCallExistedMethod) {
+    loadDll(get_mock_engine_name());
+
+    auto factory = make_std_function("CreatePluginEngine");
+    std::shared_ptr<InferenceEngine::IInferencePlugin> ptr;
+    EXPECT_NO_THROW(factory(ptr));
+}

inference-engine/tests/functional/inference_engine/ov_variable_state_test.cpp

@@ -0,0 +1,31 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <gtest/gtest.h>
+
+#include <openvino/runtime/variable_state.hpp>
+
+using namespace ::testing;
+using namespace std;
+
+TEST(VariableStateOVTests, throwsOnUninitializedReset) {
+    ov::runtime::VariableState state;
+    ASSERT_THROW(state.reset(), InferenceEngine::NotAllocated);
+}
+
+TEST(VariableStateOVTests, throwsOnUninitializedGetname) {
+    ov::runtime::VariableState state;
+    ASSERT_THROW(state.get_name(), InferenceEngine::NotAllocated);
+}
+
+TEST(VariableStateOVTests, throwsOnUninitializedGetState) {
+    ov::runtime::VariableState state;
+    ASSERT_THROW(state.get_state(), InferenceEngine::NotAllocated);
+}
+
+TEST(VariableStateOVTests, throwsOnUninitializedSetState) {
+    ov::runtime::VariableState state;
+    InferenceEngine::Blob::Ptr blob;
+    ASSERT_THROW(state.set_state(blob), InferenceEngine::NotAllocated);
+}