From ed6624c489bb6dc4daa29fed1c0e496f0dd1ab53 Mon Sep 17 00:00:00 2001 From: Anton Pankratv Date: Tue, 24 Aug 2021 07:14:11 +0300 Subject: [PATCH] Added openvino infer request API (#7151) --- .../include/openvino/runtime/common.hpp | 3 +- .../openvino/runtime/infer_request.hpp | 178 ++++++++++++++++++ .../openvino/runtime/profiling_info.hpp | 65 +++++++ .../openvino/runtime/variable_state.hpp | 81 ++++++++ .../src/cpp/ie_infer_request.cpp | 132 ++++++++++++- .../src/cpp/ie_variable_state.cpp | 31 ++- .../src/os/lin/lin_shared_object_loader.cpp | 31 +++ .../src/os/win/win_shared_object_loader.cpp | 110 +++++++++++ .../src/plugin_api/shared_object.hpp | 44 +++++ .../ov_infer_request_test.cpp | 75 ++++++++ .../ov_shared_object_test.cpp | 61 ++++++ .../ov_variable_state_test.cpp | 31 +++ 12 files changed, 839 insertions(+), 3 deletions(-) create mode 100644 inference-engine/src/inference_engine/include/openvino/runtime/infer_request.hpp create mode 100644 inference-engine/src/inference_engine/include/openvino/runtime/profiling_info.hpp create mode 100644 inference-engine/src/inference_engine/include/openvino/runtime/variable_state.hpp create mode 100644 inference-engine/src/plugin_api/shared_object.hpp create mode 100644 inference-engine/tests/functional/inference_engine/ov_infer_request_test.cpp create mode 100644 inference-engine/tests/functional/inference_engine/ov_shared_object_test.cpp create mode 100644 inference-engine/tests/functional/inference_engine/ov_variable_state_test.cpp diff --git a/inference-engine/src/inference_engine/include/openvino/runtime/common.hpp b/inference-engine/src/inference_engine/include/openvino/runtime/common.hpp index 9c0c2e93192..4cb98fa034f 100644 --- a/inference-engine/src/inference_engine/include/openvino/runtime/common.hpp +++ b/inference-engine/src/inference_engine/include/openvino/runtime/common.hpp @@ -3,12 +3,13 @@ // /** - * @brief This is a header file for the OpenVINO Runtime common aliases that depend only from external API + * @brief This is a header file for the OpenVINO Runtime common aliases and data types * * @file openvino/runtime/common.hpp */ #pragma once +#include #include #include diff --git a/inference-engine/src/inference_engine/include/openvino/runtime/infer_request.hpp b/inference-engine/src/inference_engine/include/openvino/runtime/infer_request.hpp new file mode 100644 index 00000000000..d5993d9a09c --- /dev/null +++ b/inference-engine/src/inference_engine/include/openvino/runtime/infer_request.hpp @@ -0,0 +1,178 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief A header file that provides wrapper classes for infer requests and callbacks. + * + * @file infer_request.hpp + */ +#pragma once + +#include +#include +#include + +#include "common.hpp" +#include "profiling_info.hpp" +#include "variable_state.hpp" + +namespace InferenceEngine { +class IInferRequestInternal; +class Blob; +} // namespace InferenceEngine + +namespace ov { +namespace runtime { +/** + * @brief This is an interface of asynchronous infer request + * + * It can throw exceptions safely for the application, where it is properly handled. + */ +class INFERENCE_ENGINE_API_CLASS(InferRequest) { + std::shared_ptr _so; + std::shared_ptr _impl; + + /** + * @brief Constructs InferRequest from the initialized std::shared_ptr + * @param so Plugin to use. This is required to ensure that InferRequest can work properly even if plugin object is + * destroyed. + * @param impl Initialized shared pointer + */ + InferRequest(const std::shared_ptr& so, const std::shared_ptr& impl); + friend class ExecutableNetwork; + +public: + /** + * @brief Default constructor + */ + InferRequest() = default; + + /** + * @brief Sets input/output data to infer + * + * @note Memory allocation does not happen + * @param name Name of input or output blob. + * @param data Reference to input or output blob. The type of a blob must match the network input precision and + * size. + */ + void set_blob(const std::string& name, const std::shared_ptr& data); + + /** + * @brief Gets input/output data for inference + * + * @note Memory allocation does not happen + * @param name A name of Blob to get + * @return A shared pointer to a Blob with a name @p name. If a blob is not found, an exception is thrown. + */ + std::shared_ptr get_blob(const std::string& name); + + /** + * @brief Infers specified input(s) in synchronous mode + * + * @note blocks all methods of InferRequest while request is ongoing (running or waiting in queue) + * + */ + void infer(); + + /** + * @brief Cancels inference request + */ + void cancel(); + + /** + * @brief Queries performance measures per layer to get feedback of what is the most time consuming layer + * + * @note not all plugins provide meaningful data + * @return Vector of profiling information for layers in network + */ + std::vector get_profiling_info() const; + + /** + * @brief Sets input data to infer + * + * @note Memory allocation doesn't happen + * @param inputs A reference to a map of input blobs accessed by input names. + * The type of Blob must correspond to the network input precision and size. + */ + void set_input(const std::map>& inputs); + + /** + * @brief Sets data that will contain result of the inference + * + * @note Memory allocation doesn't happen + * @param results - a reference to a map of result blobs accessed by output names. + * The type of Blob must correspond to the network output precision and size. + */ + void set_output(const std::map>& results); + + /** + * @brief Sets new batch size when dynamic batching is enabled in executable network that created this request. + * + * @param batch new batch size to be used by all the following inference calls for this request. + */ + void set_batch(const int batch); + + /** + * @brief Start inference of specified input(s) in asynchronous mode + * + * @note It returns immediately. Inference starts also immediately. + */ + void start_async(); + + /** + * @brief Waits for the result to become available. Blocks until the result + * becomes available + */ + void wait(); + + /** + * @brief Waits for the result to become available. Blocks until specified timeout has elapsed or the result + * becomes available, whichever comes first. + * + * @param timeout Maximum duration in milliseconds to block for + * @return true if inference request is ready and false otherwise + */ + bool wait_for(const std::chrono::milliseconds timeout); + + /** + * @brief Sets a callback function that will be called on success or failure of asynchronous request + * + * @param callback callback object which will be called on when inference finish. + */ + void set_callback(std::function callback); + + /** + * @brief Gets state control interface for given infer request. + * + * State control essential for recurrent networks + * @return A vector of Memory State objects + */ + std::vector query_state(); + + /** + * @brief Checks if current InferRequest object is not initialized + * @return true if current InferRequest object is not initialized, false - otherwise + */ + bool operator!() const noexcept; + + /** + * @brief Checks if current InferRequest object is initialized + * @return true if current InferRequest object is initialized, false - otherwise + */ + explicit operator bool() const noexcept; + + /** + * @brief Compares whether this request wraps the same impl underneath + * @return true if current InferRequest object doesn't wrap the same impl as the operator's arg + */ + bool operator!=(const InferRequest&) const noexcept; + + /** + * @brief Compares whether this request wraps the same impl underneath + * @return true if current InferRequest object wraps the same impl as the operator's arg + */ + bool operator==(const InferRequest&) const noexcept; +}; +} // namespace runtime +} // namespace ov diff --git a/inference-engine/src/inference_engine/include/openvino/runtime/profiling_info.hpp b/inference-engine/src/inference_engine/include/openvino/runtime/profiling_info.hpp new file mode 100644 index 00000000000..f72255071a9 --- /dev/null +++ b/inference-engine/src/inference_engine/include/openvino/runtime/profiling_info.hpp @@ -0,0 +1,65 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief This is a header file for the ProfilingInfo objects that contains performance + * metric for single node + * + * @file openvino/runtime/profiling_info.hpp + */ +#pragma once + +#include +#include + +namespace ov { +namespace runtime { +/** + * @struct ProfilingInfo + * @brief Represents basic inference profiling information per node. + * + * If the node is executed using tiling, the sum time per each tile is indicated as the total execution time. + * Due to parallel execution, the total execution time for all nodes might be greater than the total inference time. + */ +struct ProfilingInfo { + /** + * @brief Defines the general status of the node + */ + enum class Status { + NOT_RUN, //!< A node is not executed + OPTIMIZED_OUT, //!< A node is optimized out during graph optimization phase + EXECUTED //!< A node is executed + }; + + /** + * @brief Defines a node status + */ + Status status; + + /** + * @brief The absolute time in microseconds that the node ran (in total) + */ + std::chrono::microseconds real_time; + /** + * @brief The net host cpu time that the node ran + */ + std::chrono::microseconds cpu_time; + + /** + * @brief A name of node + */ + std::string node_name; + + /** + * @brief An execution type of unit + */ + std::string exec_type; + + /** + * @brief A node type + */ + std::string node_type; +}; +} // namespace runtime +} // namespace ov \ No newline at end of file diff --git a/inference-engine/src/inference_engine/include/openvino/runtime/variable_state.hpp b/inference-engine/src/inference_engine/include/openvino/runtime/variable_state.hpp new file mode 100644 index 00000000000..e15679b118b --- /dev/null +++ b/inference-engine/src/inference_engine/include/openvino/runtime/variable_state.hpp @@ -0,0 +1,81 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief A header file that provides VariableState + * + * @file variable_state.hpp + */ + +#pragma once + +#include +#include + +#include +#include + +#include "common.hpp" + +namespace InferenceEngine { +class IVariableStateInternal; +class Blob; +} // namespace InferenceEngine + +namespace ov { +namespace runtime { + +class SharedObject; +class InferRequest; + +/** + * @brief VariableState class + */ +class INFERENCE_ENGINE_API_CLASS(VariableState) { + std::shared_ptr _so; + std::shared_ptr _impl; + + /** + * @brief Constructs VariableState from the initialized std::shared_ptr + * @param impl Initialized shared pointer + * @param so Optional: Plugin to use. This is required to ensure that VariableState can work properly even if plugin + * object is destroyed. + */ + VariableState(const std::shared_ptr& so, const std::shared_ptr& impl); + + friend class ov::runtime::InferRequest; + +public: + /** + * @brief Default constructor + */ + VariableState() = default; + + /** + * @brief Reset internal variable state for relevant infer request, + * to a value specified as default for according ReadValue node + */ + void reset(); + + /** + * @brief Gets name of current variable state, if length of array is not enough name is truncated by len, null + * terminator is inserted as well. As variable state name `variable_id` from according `ReadValue` used. + * @return A string representing a state name + */ + std::string get_name() const; + + /** + * @brief Returns the value of the variable state. + * @return A blob representing a state + */ + std::shared_ptr get_state() const; + + /** + * @brief Sets the new state for the next inference. + * @param state The current state to set + */ + void set_state(const std::shared_ptr& state); +}; +} // namespace runtime +} // namespace ov diff --git a/inference-engine/src/inference_engine/src/cpp/ie_infer_request.cpp b/inference-engine/src/inference_engine/src/cpp/ie_infer_request.cpp index 135cf2d3391..11937e6309d 100644 --- a/inference-engine/src/inference_engine/src/cpp/ie_infer_request.cpp +++ b/inference-engine/src/inference_engine/src/cpp/ie_infer_request.cpp @@ -10,8 +10,10 @@ #include "cpp/exception2status.hpp" #include "cpp_interfaces/interface/ie_iinfer_request_internal.hpp" +#include "details/ie_so_loader.h" #include "ie_infer_async_request_base.hpp" #include "ie_remote_context.hpp" +#include "openvino/runtime/infer_request.hpp" namespace InferenceEngine { @@ -21,7 +23,7 @@ namespace InferenceEngine { try { \ __VA_ARGS__ \ } catch (...) { \ - details::Rethrow(); \ + ::InferenceEngine::details::Rethrow(); \ } InferRequest::InferRequest(const details::SharedObjectLoader& so, const IInferRequestInternal::Ptr& impl) @@ -190,3 +192,131 @@ bool InferRequest::operator==(const InferRequest& r) const noexcept { } } // namespace InferenceEngine + +namespace ov { +namespace runtime { + +InferRequest::InferRequest(const std::shared_ptr& so, const ie::IInferRequestInternal::Ptr& impl) + : _so{so}, + _impl{impl} { + IE_ASSERT(_impl != nullptr); +} + +void InferRequest::set_blob(const std::string& name, const ie::Blob::Ptr& data) { + INFER_REQ_CALL_STATEMENT(_impl->SetBlob(name, data);) +} + +ie::Blob::Ptr InferRequest::get_blob(const std::string& name) { + ie::Blob::Ptr blobPtr; + INFER_REQ_CALL_STATEMENT(blobPtr = _impl->GetBlob(name);) + std::string error = "Internal error: blob with name `" + name + "` is not allocated!"; + const bool remoteBlobPassed = blobPtr->is(); + if (blobPtr == nullptr) + IE_THROW() << error; + if (!remoteBlobPassed && blobPtr->buffer() == nullptr) + IE_THROW() << error; + return blobPtr; +} + +void InferRequest::infer() { + INFER_REQ_CALL_STATEMENT(_impl->Infer();) +} + +void InferRequest::cancel() { + INFER_REQ_CALL_STATEMENT(_impl->Cancel();) +} + +std::vector InferRequest::get_profiling_info() const { + INFER_REQ_CALL_STATEMENT({ + auto ieInfos = _impl->GetPerformanceCounts(); + std::vector infos; + infos.reserve(ieInfos.size()); + while (!ieInfos.empty()) { + auto itIeInfo = std::min_element( + std::begin(ieInfos), + std::end(ieInfos), + [](const decltype(ieInfos)::value_type& lhs, const decltype(ieInfos)::value_type& rhs) { + return lhs.second.execution_index < rhs.second.execution_index; + }); + IE_ASSERT(itIeInfo != ieInfos.end()); + auto& ieInfo = itIeInfo->second; + infos.push_back(ProfilingInfo{}); + auto& info = infos.back(); + switch (ieInfo.status) { + case ie::InferenceEngineProfileInfo::NOT_RUN: + info.status = ProfilingInfo::Status::NOT_RUN; + break; + case ie::InferenceEngineProfileInfo::OPTIMIZED_OUT: + info.status = ProfilingInfo::Status::OPTIMIZED_OUT; + break; + case ie::InferenceEngineProfileInfo::EXECUTED: + info.status = ProfilingInfo::Status::OPTIMIZED_OUT; + break; + } + info.real_time = std::chrono::microseconds{ieInfo.realTime_uSec}; + info.cpu_time = std::chrono::microseconds{ieInfo.cpu_uSec}; + info.node_name = itIeInfo->first; + info.exec_type = std::string{ieInfo.exec_type}; + info.node_type = std::string{ieInfo.layer_type}; + ieInfos.erase(itIeInfo); + } + return infos; + }) +} + +void InferRequest::set_input(const ie::BlobMap& inputs) { + INFER_REQ_CALL_STATEMENT(for (auto&& input : inputs) { _impl->SetBlob(input.first, input.second); }) +} + +void InferRequest::set_output(const ie::BlobMap& results) { + INFER_REQ_CALL_STATEMENT(for (auto&& result : results) { _impl->SetBlob(result.first, result.second); }) +} + +void InferRequest::set_batch(const int batch) { + INFER_REQ_CALL_STATEMENT(_impl->SetBatch(batch);) +} + +void InferRequest::start_async() { + INFER_REQ_CALL_STATEMENT(_impl->StartAsync();) +} + +void InferRequest::wait() { + INFER_REQ_CALL_STATEMENT(_impl->Wait(ie::InferRequest::RESULT_READY);) +} + +bool InferRequest::wait_for(const std::chrono::milliseconds timeout) { + INFER_REQ_CALL_STATEMENT(return _impl->Wait(timeout.count()) == ie::OK;) +} + +void InferRequest::set_callback(std::function callback) { + INFER_REQ_CALL_STATEMENT(_impl->SetCallback(std::move(callback));) +} + +std::vector InferRequest::query_state() { + std::vector variable_states; + INFER_REQ_CALL_STATEMENT({ + for (auto&& state : _impl->QueryState()) { + variable_states.emplace_back(VariableState{_so, state}); + } + }) + return variable_states; +} + +bool InferRequest::operator!() const noexcept { + return !_impl; +} + +InferRequest::operator bool() const noexcept { + return (!!_impl); +} + +bool InferRequest::operator!=(const InferRequest& r) const noexcept { + return !(r == *this); +} + +bool InferRequest::operator==(const InferRequest& r) const noexcept { + return r._impl == _impl; +} + +} // namespace runtime +} // namespace ov \ No newline at end of file diff --git a/inference-engine/src/inference_engine/src/cpp/ie_variable_state.cpp b/inference-engine/src/inference_engine/src/cpp/ie_variable_state.cpp index ff8547f13b3..e9aeb57d720 100644 --- a/inference-engine/src/inference_engine/src/cpp/ie_variable_state.cpp +++ b/inference-engine/src/inference_engine/src/cpp/ie_variable_state.cpp @@ -6,6 +6,7 @@ #include "cpp_interfaces/interface/ie_ivariable_state_internal.hpp" #include "details/ie_so_loader.h" #include "exception2status.hpp" +#include "openvino/runtime/variable_state.hpp" #define VARIABLE_CALL_STATEMENT(...) \ if (_impl == nullptr) \ @@ -13,7 +14,7 @@ try { \ __VA_ARGS__; \ } catch (...) { \ - details::Rethrow(); \ + ::InferenceEngine::details::Rethrow(); \ } namespace InferenceEngine { @@ -44,3 +45,31 @@ void VariableState::SetState(Blob::Ptr state) { } } // namespace InferenceEngine + +namespace ov { +namespace runtime { + +VariableState::VariableState(const std::shared_ptr& so, const ie::IVariableStateInternal::Ptr& impl) + : _so{so}, + _impl{impl} { + IE_ASSERT(_impl != nullptr); +} + +void VariableState::reset() { + VARIABLE_CALL_STATEMENT(_impl->Reset()); +} + +std::string VariableState::get_name() const { + VARIABLE_CALL_STATEMENT(return _impl->GetName()); +} + +ie::Blob::CPtr VariableState::get_state() const { + VARIABLE_CALL_STATEMENT(return _impl->GetState()); +} + +void VariableState::set_state(const ie::Blob::Ptr& state) { + VARIABLE_CALL_STATEMENT(_impl->SetState(state)); +} + +} // namespace runtime +} // namespace ov diff --git a/inference-engine/src/inference_engine/src/os/lin/lin_shared_object_loader.cpp b/inference-engine/src/inference_engine/src/os/lin/lin_shared_object_loader.cpp index 6d3cfa87e31..1961c1a7130 100644 --- a/inference-engine/src/inference_engine/src/os/lin/lin_shared_object_loader.cpp +++ b/inference-engine/src/inference_engine/src/os/lin/lin_shared_object_loader.cpp @@ -8,6 +8,7 @@ #include "details/ie_so_loader.h" #include "file_utils.h" +#include "shared_object.hpp" namespace InferenceEngine { namespace details { @@ -71,3 +72,33 @@ void* SharedObjectLoader::get_symbol(const char* symbolName) const { } // namespace details } // namespace InferenceEngine + +namespace ov { +namespace runtime { +SharedObject::SharedObject(const char* path) { + shared_object = dlopen(path, RTLD_NOW); + + if (shared_object == nullptr) + IE_THROW() << "Cannot load library '" << path << "': " << dlerror(); +} + +#ifdef ENABLE_UNICODE_PATH_SUPPORT +SharedObject::SharedObject(const wchar_t* path) : SharedObject(FileUtils::wStringtoMBCSstringChar(path).c_str()) {} +#endif // ENABLE_UNICODE_PATH_SUPPORT + +SharedObject::~SharedObject() { + if (0 != dlclose(shared_object)) { + std::cerr << "dlclose failed: " << dlerror() << std::endl; + } +} + +void* SharedObject::get_symbol(const char* symbolName) const { + void* procAddr = nullptr; + + procAddr = dlsym(shared_object, symbolName); + if (procAddr == nullptr) + IE_THROW(NotFound) << "dlSym cannot locate method '" << symbolName << "': " << dlerror(); + return procAddr; +} +} // namespace runtime +} // namespace ov diff --git a/inference-engine/src/inference_engine/src/os/win/win_shared_object_loader.cpp b/inference-engine/src/inference_engine/src/os/win/win_shared_object_loader.cpp index 31c148ec111..7896b56e5d2 100644 --- a/inference-engine/src/inference_engine/src/os/win/win_shared_object_loader.cpp +++ b/inference-engine/src/inference_engine/src/os/win/win_shared_object_loader.cpp @@ -5,6 +5,7 @@ #include "ie_common.h" #include "details/ie_so_loader.h" #include "file_utils.h" +#include "shared_object.hpp" // // LoadLibraryA, LoadLibraryW: @@ -274,3 +275,112 @@ void* SharedObjectLoader::get_symbol(const char* symbolName) const { } // namespace details } // namespace InferenceEngine + + +namespace ov { +namespace runtime { +SharedObject::SharedObject(const char* path) { + using GetDllDirectoryA_Fnc = DWORD(*)(DWORD, LPSTR); + GetDllDirectoryA_Fnc IEGetDllDirectoryA = nullptr; + if (HMODULE hm = GetModuleHandleW(L"kernel32.dll")) { + IEGetDllDirectoryA = reinterpret_cast(GetProcAddress(hm, "GetDllDirectoryA")); + } +#if !WINAPI_PARTITION_SYSTEM + // ExcludeCurrentDirectory + if (IEGetDllDirectoryA && IEGetDllDirectoryA(0, NULL) <= 1) { + SetDllDirectoryA(""); + } + // LoadPluginFromDirectory + if (IEGetDllDirectoryA) { + DWORD nBufferLength = IEGetDllDirectoryA(0, NULL); + std::vector lpBuffer(nBufferLength); + IEGetDllDirectoryA(nBufferLength, &lpBuffer.front()); + + // GetDirname + auto dirname = [&] { + auto pos = strchr(path, '\\'); + if (pos == nullptr) { + return std::string{path}; + } + std::string original(path); + original[pos - path] = 0; + return original; + } (); + + SetDllDirectoryA(dirname.c_str()); + shared_object = LoadLibraryA(path); + + SetDllDirectoryA(&lpBuffer.front()); + } +#endif + if (!shared_object) { + shared_object = LoadLibraryA(path); + } + if (!shared_object) { + char cwd[1024]; + IE_THROW() << "Cannot load library '" << path << "': " << GetLastError() + << " from cwd: " << _getcwd(cwd, sizeof(cwd)); + } +} + +#ifdef ENABLE_UNICODE_PATH_SUPPORT +SharedObject::SharedObject(const wchar_t* path) { + using GetDllDirectoryW_Fnc = DWORD(*)(DWORD, LPWSTR); + static GetDllDirectoryW_Fnc IEGetDllDirectoryW = nullptr; + if (HMODULE hm = GetModuleHandleW(L"kernel32.dll")) { + IEGetDllDirectoryW = reinterpret_cast(GetProcAddress(hm, "GetDllDirectoryW")); + } + // ExcludeCurrentDirectory +#if !WINAPI_PARTITION_SYSTEM + if (IEGetDllDirectoryW && IEGetDllDirectoryW(0, NULL) <= 1) { + SetDllDirectoryW(L""); + } + if (IEGetDllDirectoryW) { + DWORD nBufferLength = IEGetDllDirectoryW(0, NULL); + std::vector lpBuffer(nBufferLength); + IEGetDllDirectoryW(nBufferLength, &lpBuffer.front()); + + auto dirname = [&] { + auto pos = wcsrchr(path, '\\'); + if (pos == nullptr) { + return std::wstring{path}; + } + std::wstring original(path); + original[pos - path] = 0; + return original; + } (); + SetDllDirectoryW(dirname.c_str()); + shared_object = LoadLibraryW(path); + + SetDllDirectoryW(&lpBuffer.front()); + } +#endif + if (!shared_object) { + shared_object = LoadLibraryW(path); + } + if (!shared_object) { + char cwd[1024]; + IE_THROW() << "Cannot load library '" << FileUtils::wStringtoMBCSstringChar(std::wstring(path)) << "': " << GetLastError() + << " from cwd: " << _getcwd(cwd, sizeof(cwd)); + } +} +#endif + +SharedObject::~SharedObject() { + FreeLibrary(reinterpret_cast(shared_object)); +} + +void* SharedObject::get_symbol(const char* symbolName) const { + if (!shared_object) { + IE_THROW() << "Cannot get '" << symbolName << "' content from unknown library!"; + } + auto procAddr = reinterpret_cast(GetProcAddress( + reinterpret_cast(const_cast(shared_object)), symbolName)); + if (procAddr == nullptr) + IE_THROW(NotFound) + << "GetProcAddress cannot locate method '" << symbolName << "': " << GetLastError(); + + return procAddr; +} +} // namespace runtime +} // namespace ov diff --git a/inference-engine/src/plugin_api/shared_object.hpp b/inference-engine/src/plugin_api/shared_object.hpp new file mode 100644 index 00000000000..de2523375c7 --- /dev/null +++ b/inference-engine/src/plugin_api/shared_object.hpp @@ -0,0 +1,44 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief A header file for definition of abstraction over platform specific shared objects + * @file ie_system_conf.h + */ + +#pragma once + +#include "ie_api.h" + +namespace ov { +namespace runtime { +struct INFERENCE_ENGINE_API_CLASS(SharedObject) { + void* shared_object = nullptr; + + /** + * @brief Loads a library with the name specified. + * @param path Full or relative path to the plugin library + */ + explicit SharedObject(const char* path); + +#ifdef ENABLE_UNICODE_PATH_SUPPORT + /** + * @brief Loads a library with the wide char name specified. + * @param path Full or relative path to the plugin library + */ + explicit SharedObject(const wchar_t* path); +#endif // ENABLE_UNICODE_PATH_SUPPORT + + ~SharedObject(); + + /** + * @brief Searches for a function symbol in the loaded module + * @param symbolName Name of the function to find + * @return A pointer to the function if found + * @throws Exception if the function is not found + */ + void* get_symbol(const char* symbolName) const; +}; +} // namespace runtime +} // namespace ov diff --git a/inference-engine/tests/functional/inference_engine/ov_infer_request_test.cpp b/inference-engine/tests/functional/inference_engine/ov_infer_request_test.cpp new file mode 100644 index 00000000000..1cfdb17941c --- /dev/null +++ b/inference-engine/tests/functional/inference_engine/ov_infer_request_test.cpp @@ -0,0 +1,75 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include +#include + +using namespace ::testing; +using namespace std; +using namespace InferenceEngine; +using namespace InferenceEngine::details; + + +TEST(InferRequestOVTests, throwsOnUninitializedSetBlob) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.set_blob({}, {}), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedGetBlob) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.get_blob({}), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedInfer) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.infer(), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedGetPerformanceCounts) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.get_profiling_info(), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedSetInput) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.set_input({{}}), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedSetOutput) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.set_output({{}}), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedSetBatch) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.set_batch({}), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedStartAsync) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.start_async(), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedWait) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.wait(), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedWaitFor) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.wait_for({}), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedSetCompletionCallback) { + ov::runtime::InferRequest req; + std::function f; + ASSERT_THROW(req.set_callback(f), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedQueryState) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.query_state(), InferenceEngine::NotAllocated); +} diff --git a/inference-engine/tests/functional/inference_engine/ov_shared_object_test.cpp b/inference-engine/tests/functional/inference_engine/ov_shared_object_test.cpp new file mode 100644 index 00000000000..a396df7afc9 --- /dev/null +++ b/inference-engine/tests/functional/inference_engine/ov_shared_object_test.cpp @@ -0,0 +1,61 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include +#include "shared_object.hpp" +#include + +using namespace ::testing; +using namespace std; + +class SharedObjectOVTests: public ::testing::Test { +protected: + std::string get_mock_engine_name() { + return FileUtils::makePluginLibraryName(InferenceEngine::getIELibraryPath(), + std::string("mock_engine") + IE_BUILD_POSTFIX); + } + + void loadDll(const string &libraryName) { + sharedObject.reset(new ov::runtime::SharedObject(libraryName.c_str())); + } + unique_ptr sharedObject; + + using CreateF = void(std::shared_ptr&); + + std::function make_std_function(const std::string& functionName) { + std::function ptr(reinterpret_cast(sharedObject->get_symbol(functionName.c_str()))); + return ptr; + } +}; + +TEST_F(SharedObjectOVTests, canLoadExistedPlugin) { + loadDll(get_mock_engine_name()); + EXPECT_NE(nullptr, sharedObject.get()); +} + +TEST_F(SharedObjectOVTests, loaderThrowsIfNoPlugin) { + EXPECT_THROW(loadDll("wrong_name"), InferenceEngine::Exception); +} + +TEST_F(SharedObjectOVTests, canFindExistedMethod) { + loadDll(get_mock_engine_name()); + + auto factory = make_std_function("CreatePluginEngine"); + EXPECT_NE(nullptr, factory); +} + +TEST_F(SharedObjectOVTests, throwIfMethodNofFoundInLibrary) { + loadDll(get_mock_engine_name()); + EXPECT_THROW(make_std_function("wrong_function"), InferenceEngine::Exception); +} + +TEST_F(SharedObjectOVTests, canCallExistedMethod) { + loadDll(get_mock_engine_name()); + + auto factory = make_std_function("CreatePluginEngine"); + std::shared_ptr ptr; + EXPECT_NO_THROW(factory(ptr)); +} diff --git a/inference-engine/tests/functional/inference_engine/ov_variable_state_test.cpp b/inference-engine/tests/functional/inference_engine/ov_variable_state_test.cpp new file mode 100644 index 00000000000..f9233e5eebc --- /dev/null +++ b/inference-engine/tests/functional/inference_engine/ov_variable_state_test.cpp @@ -0,0 +1,31 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include + +using namespace ::testing; +using namespace std; + +TEST(VariableStateOVTests, throwsOnUninitializedReset) { + ov::runtime::VariableState state; + ASSERT_THROW(state.reset(), InferenceEngine::NotAllocated); +} + +TEST(VariableStateOVTests, throwsOnUninitializedGetname) { + ov::runtime::VariableState state; + ASSERT_THROW(state.get_name(), InferenceEngine::NotAllocated); +} + +TEST(VariableStateOVTests, throwsOnUninitializedGetState) { + ov::runtime::VariableState state; + ASSERT_THROW(state.get_state(), InferenceEngine::NotAllocated); +} + +TEST(VariableStateOVTests, throwsOnUninitializedSetState) { + ov::runtime::VariableState state; + InferenceEngine::Blob::Ptr blob; + ASSERT_THROW(state.set_state(blob), InferenceEngine::NotAllocated); +}