IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Moved executor manager to new API (#15871)

Browse Source 
* Moved executor manager to new API

* Fixed template plugin build

* Remove setTBBFlag API

* Fixed export

* Added new files

* Revert "Added new files"

This reverts commit 981c3c863f.

* Fixed template plugin tests

* Remove redundant wrapper

* Remove wrappers for executor manager

* Fixed build
This commit is contained in:
Ilya Churaev 2023-02-22 20:19:35 +04:00 committed by GitHub
parent 98392a043b
commit 27ea9eab32
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 345 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/inference/CMakeLists.txt
View File

@ -14,6 +14,7 @@ file (GLOB LIBRARY_SRC
${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/*.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/dev/*.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/dev/preprocessing/*.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/dev/threading/*.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/threading/*.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/*.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/cpp_interfaces/interface/*.cpp

14
src/inference/dev_api/openvino/runtime/iplugin.hpp
View File

@ -19,7 +19,7 @@
#include "openvino/runtime/icompiled_model.hpp"
#include "openvino/runtime/icore.hpp"
#include "openvino/runtime/remote_context.hpp"
#include "threading/ie_executor_manager.hpp"
#include "openvino/runtime/threading/executor_manager.hpp"
namespace InferenceEngine {
@ -188,7 +188,7 @@ public:
* @brief Gets reference to tasks execution manager
* @return Reference to ExecutorManager interface
*/
const std::shared_ptr<InferenceEngine::ExecutorManager>& get_executor_manager() const;
const std::shared_ptr<ov::ExecutorManager>& get_executor_manager() const;
~IPlugin() = default;
@ -198,11 +198,11 @@ protected:
private:
friend ::InferenceEngine::IPluginWrapper;
std::string m_plugin_name; //!< A device name that plugins enables
std::weak_ptr<ov::ICore> m_core; //!< A pointer to ICore interface
std::shared_ptr<InferenceEngine::ExecutorManager> m_executor_manager; //!< A tasks execution manager
ov::Version m_version; //!< Member contains plugin version
bool m_is_new_api; //!< A flag which shows used API
std::string m_plugin_name; //!< A device name that plugins enables
std::weak_ptr<ov::ICore> m_core; //!< A pointer to ICore interface
std::shared_ptr<ov::ExecutorManager> m_executor_manager; //!< A tasks execution manager
ov::Version m_version; //!< Member contains plugin version
bool m_is_new_api; //!< A flag which shows used API
};
} // namespace ov

77
src/inference/dev_api/openvino/runtime/threading/executor_manager.hpp Normal file
View File

@ -0,0 +1,77 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
/**
* @brief OpenVINO Runtime Executor Manager
* @file openvino/runtime/threading/executor_manager.hpp
*/
#pragma once
#include "openvino/runtime/common.hpp"
#include "threading/ie_istreams_executor.hpp"
#include "threading/ie_itask_executor.hpp"
namespace ov {
/**
* @interface ExecutorManager
* @brief Interface for tasks execution manager.
* This is global point for getting task executor objects by string id.
* It's necessary in multiple asynchronous requests for having unique executors to avoid oversubscription.
* E.g. There 2 task executors for CPU device: one - in FPGA, another - in OneDNN. Parallel execution both of them leads
* to not optimal CPU usage. More efficient to run the corresponding tasks one by one via single executor.
* @ingroup ov_dev_api_threading
*/
class OPENVINO_RUNTIME_API ExecutorManager {
public:
/**
* @brief Returns executor by unique identificator
* @param id An unique identificator of device (Usually string representation of TargetDevice)
* @return A shared pointer to existing or newly ITaskExecutor
*/
virtual InferenceEngine::ITaskExecutor::Ptr get_executor(const std::string& id) = 0;
/**
* @brief Returns idle cpu streams executor
*
* @param config Streams executor config
*
* @return pointer to streams executor config
*/
virtual InferenceEngine::IStreamsExecutor::Ptr get_idle_cpu_streams_executor(
const InferenceEngine::IStreamsExecutor::Config& config) = 0;
/**
* @brief Allows to configure executor manager
*
* @param properties map with configuration
*/
virtual void set_property(const ov::AnyMap& properties) = 0;
/**
* @brief Returns configuration
*
* @param name property name
*
* @return Property value
*/
virtual ov::Any get_property(const std::string& name) const = 0;
/**
* @cond
*/
virtual size_t get_executors_number() const = 0;
virtual size_t get_idle_cpu_streams_executors_number() const = 0;
virtual void clear(const std::string& id = {}) = 0;
/**
* @endcond
*/
virtual ~ExecutorManager() = default;
};
OPENVINO_API std::shared_ptr<ExecutorManager> executor_manager();
} // namespace ov

15
src/inference/dev_api/threading/ie_executor_manager.hpp
View File

@ -18,8 +18,16 @@
#include "threading/ie_istreams_executor.hpp"
#include "threading/ie_itask_executor.hpp"
namespace ov {
class ExecutorManager;
}
namespace InferenceEngine {
class IPluginWrapper;
/**
* @interface ExecutorManager
* @brief Interface for tasks execution manager.
@ -76,8 +84,15 @@ public:
*/
virtual void setTbbFlag(bool flag) = 0;
virtual bool getTbbFlag() = 0;
private:
virtual std::shared_ptr<ov::ExecutorManager> get_ov_manager() const = 0;
friend class IPluginWrapper;
};
INFERENCE_ENGINE_API_CPP(ExecutorManager::Ptr) executorManager();
std::shared_ptr<InferenceEngine::ExecutorManager> create_old_manager(
const std::shared_ptr<ov::ExecutorManager>& manager);
} // namespace InferenceEngine

4
src/inference/src/dev/converter_utils.cpp
View File

@ -34,8 +34,10 @@
#include "openvino/runtime/profiling_info.hpp"
#include "openvino/runtime/remote_context.hpp"
#include "openvino/runtime/tensor.hpp"
#include "openvino/runtime/threading/executor_manager.hpp"
#include "openvino/runtime/variable_state.hpp"
#include "so_ptr.hpp"
#include "threading/ie_executor_manager.hpp"
#include "transformations/utils/utils.hpp"
namespace {
@ -221,7 +223,7 @@ public:
version.description = ver.description;
SetVersion(version);
_isNewAPI = plugin->is_new_api();
_executorManager = plugin->get_executor_manager();
_executorManager = InferenceEngine::create_old_manager(plugin->get_executor_manager());
}
std::string GetName() const noexcept override {
return m_plugin->get_device_name();

7
src/inference/src/dev/core_impl.cpp
View File

@ -28,6 +28,7 @@
#include "openvino/pass/manager.hpp"
#include "openvino/runtime/icompiled_model.hpp"
#include "openvino/runtime/remote_context.hpp"
#include "openvino/runtime/threading/executor_manager.hpp"
#include "openvino/util/common_util.hpp"
#include "openvino/util/shared_object.hpp"
#include "preprocessing/preprocessing.hpp"
@ -57,7 +58,7 @@ void stripDeviceName(std::string& device, const std::string& substr) {
ov::CoreImpl::CoreImpl(bool _newAPI) : m_new_api(_newAPI) {
add_mutex(""); // Register global mutex
executorManagerPtr = InferenceEngine::executorManager();
m_executor_manager = ov::executor_manager();
for (const auto& it : ov::get_available_opsets()) {
opsetNames.insert(it.first);
}
@ -632,7 +633,7 @@ void ov::CoreImpl::set_property(const std::string& device_name, const AnyMap& pr
ov::Any ov::CoreImpl::get_property_for_core(const std::string& name) const {
if (name == ov::force_tbb_terminate.name()) {
const auto flag = InferenceEngine::executorManager()->getTbbFlag();
const auto flag = ov::executor_manager()->get_property(name).as<bool>();
return decltype(ov::force_tbb_terminate)::value_type(flag);
} else if (name == ov::cache_dir.name()) {
return ov::Any(coreConfig.get_cache_dir());
@ -993,7 +994,7 @@ void ov::CoreImpl::CoreConfig::set_and_update(ov::AnyMap& config) {
it = config.find(ov::force_tbb_terminate.name());
if (it != config.end()) {
auto flag = it->second.as<std::string>() == CONFIG_VALUE(YES) ? true : false;
InferenceEngine::executorManager()->setTbbFlag(flag);
ov::executor_manager()->set_property({{it->first, flag}});
config.erase(it);
}

4
src/inference/src/dev/core_impl.hpp
View File

@ -21,7 +21,7 @@
#include "openvino/core/version.hpp"
#include "openvino/runtime/common.hpp"
#include "openvino/runtime/icompiled_model.hpp"
#include "threading/ie_executor_manager.hpp"
#include "openvino/runtime/threading/executor_manager.hpp"
#ifdef OPENVINO_STATIC_LIBRARY
# include "ie_plugins.hpp"
@ -162,7 +162,7 @@ private:
}
};
InferenceEngine::ExecutorManager::Ptr executorManagerPtr;
std::shared_ptr<ov::ExecutorManager> m_executor_manager;
mutable std::unordered_set<std::string> opsetNames;
// TODO: make extensions to be optional with conditional compilation
mutable std::vector<InferenceEngine::IExtensionPtr> extensions;

4
src/inference/src/dev/iplugin.cpp
View File

@ -4,7 +4,7 @@
#include "openvino/runtime/iplugin.hpp"
ov::IPlugin::IPlugin() : m_executor_manager(InferenceEngine::executorManager()), m_is_new_api(true) {}
ov::IPlugin::IPlugin() : m_executor_manager(ov::executor_manager()), m_is_new_api(true) {}
void ov::IPlugin::set_version(const ov::Version& version) {
m_version = version;
@ -42,7 +42,7 @@ bool ov::IPlugin::is_new_api() const {
return m_is_new_api;
}
const std::shared_ptr<InferenceEngine::ExecutorManager>& ov::IPlugin::get_executor_manager() const {
const std::shared_ptr<ov::ExecutorManager>& ov::IPlugin::get_executor_manager() const {
return m_executor_manager;
}

3
src/inference/src/dev/iplugin_wrapper.cpp
View File

@ -9,6 +9,7 @@
#include "any_copy.hpp"
#include "dev/converter_utils.hpp"
#include "ie_icore.hpp"
#include "threading/ie_executor_manager.hpp"
namespace InferenceEngine {
@ -20,7 +21,7 @@ IPluginWrapper::IPluginWrapper(const std::shared_ptr<InferenceEngine::IInference
m_plugin_name = m_old_plugin->GetName();
m_is_new_api = m_old_plugin->IsNewAPI();
m_core = m_old_plugin->GetCore();
m_executor_manager = m_old_plugin->executorManager();
m_executor_manager = m_old_plugin->executorManager()->get_ov_manager();
}
const std::shared_ptr<InferenceEngine::IExecutableNetworkInternal>& IPluginWrapper::update_exec_network(

208
src/inference/src/dev/threading/executor_manager.cpp Normal file
View File

@ -0,0 +1,208 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "openvino/runtime/threading/executor_manager.hpp"
#include "openvino/core/parallel.hpp"
#include "openvino/runtime/properties.hpp"
#include "threading/ie_cpu_streams_executor.hpp"
#if OV_THREAD == OV_THREAD_TBB || OV_THREAD == OV_THREAD_TBB_AUTO
# if (TBB_INTERFACE_VERSION < 12000)
# include <tbb/task_scheduler_init.h>
# else
# include <oneapi/tbb/global_control.h>
# endif
#endif
#include <memory>
#include <mutex>
#include <string>
#include <utility>
namespace ov {
namespace {
class ExecutorManagerImpl : public ExecutorManager {
public:
~ExecutorManagerImpl();
InferenceEngine::ITaskExecutor::Ptr get_executor(const std::string& id) override;
InferenceEngine::IStreamsExecutor::Ptr get_idle_cpu_streams_executor(
const InferenceEngine::IStreamsExecutor::Config& config) override;
size_t get_executors_number() const override;
size_t get_idle_cpu_streams_executors_number() const override;
void clear(const std::string& id = {}) override;
void set_property(const ov::AnyMap& properties) override;
ov::Any get_property(const std::string& name) const override;
private:
void reset_tbb();
std::unordered_map<std::string, InferenceEngine::ITaskExecutor::Ptr> executors;
std::vector<std::pair<InferenceEngine::IStreamsExecutor::Config, InferenceEngine::IStreamsExecutor::Ptr>>
cpuStreamsExecutors;
mutable std::mutex streamExecutorMutex;
mutable std::mutex taskExecutorMutex;
bool tbbTerminateFlag = false;
mutable std::mutex global_mutex;
bool tbbThreadsCreated = false;
#if OV_THREAD == OV_THREAD_TBB || OV_THREAD == OV_THREAD_TBB_AUTO
# if (TBB_INTERFACE_VERSION < 12000)
std::shared_ptr<tbb::task_scheduler_init> tbbTaskScheduler = nullptr;
# else
std::shared_ptr<oneapi::tbb::task_scheduler_handle> tbbTaskScheduler = nullptr;
# endif
#endif
};
} // namespace
ExecutorManagerImpl::~ExecutorManagerImpl() {
reset_tbb();
}
void ExecutorManagerImpl::set_property(const ov::AnyMap& properties) {
std::lock_guard<std::mutex> guard(global_mutex);
for (const auto& it : properties) {
if (it.first == ov::force_tbb_terminate.name()) {
tbbTerminateFlag = it.second.as<bool>();
#if OV_THREAD == OV_THREAD_TBB || OV_THREAD == OV_THREAD_TBB_AUTO
if (tbbTerminateFlag) {
if (!tbbTaskScheduler) {
# if (TBB_INTERFACE_VERSION < 12000)
tbbTaskScheduler = std::make_shared<tbb::task_scheduler_init>();
# elif (TBB_INTERFACE_VERSION < 12060)
tbbTaskScheduler =
std::make_shared<oneapi::tbb::task_scheduler_handle>(oneapi::tbb::task_scheduler_handle::get());
# else
tbbTaskScheduler = std::make_shared<oneapi::tbb::task_scheduler_handle>(tbb::attach{});
# endif
}
} else {
tbbTaskScheduler = nullptr;
}
#endif
}
}
}
ov::Any ExecutorManagerImpl::get_property(const std::string& name) const {
std::lock_guard<std::mutex> guard(global_mutex);
if (name == ov::force_tbb_terminate.name()) {
return tbbTerminateFlag;
}
OPENVINO_UNREACHABLE("Property ", name, " is not supported.");
}
void ExecutorManagerImpl::reset_tbb() {
std::lock_guard<std::mutex> guard(global_mutex);
if (tbbTerminateFlag) {
#if OV_THREAD == OV_THREAD_TBB || OV_THREAD == OV_THREAD_TBB_AUTO
if (tbbTaskScheduler && tbbThreadsCreated) {
# if (TBB_INTERFACE_VERSION < 12000)
tbbTaskScheduler->terminate();
# else
tbb::finalize(*tbbTaskScheduler, std::nothrow);
# endif
}
tbbThreadsCreated = false;
tbbTaskScheduler = nullptr;
#endif
tbbTerminateFlag = false;
}
}
InferenceEngine::ITaskExecutor::Ptr ExecutorManagerImpl::get_executor(const std::string& id) {
std::lock_guard<std::mutex> guard(taskExecutorMutex);
auto foundEntry = executors.find(id);
if (foundEntry == executors.end()) {
auto newExec =
std::make_shared<InferenceEngine::CPUStreamsExecutor>(InferenceEngine::IStreamsExecutor::Config{id});
tbbThreadsCreated = true;
executors[id] = newExec;
return newExec;
}
return foundEntry->second;
}
InferenceEngine::IStreamsExecutor::Ptr ExecutorManagerImpl::get_idle_cpu_streams_executor(
const InferenceEngine::IStreamsExecutor::Config& config) {
std::lock_guard<std::mutex> guard(streamExecutorMutex);
for (const auto& it : cpuStreamsExecutors) {
const auto& executor = it.second;
if (executor.use_count() != 1)
continue;
const auto& executorConfig = it.first;
if (executorConfig._name == config._name && executorConfig._streams == config._streams &&
executorConfig._threadsPerStream == config._threadsPerStream &&
executorConfig._threadBindingType == config._threadBindingType &&
executorConfig._threadBindingStep == config._threadBindingStep &&
executorConfig._threadBindingOffset == config._threadBindingOffset)
if (executorConfig._threadBindingType !=
InferenceEngine::IStreamsExecutor::ThreadBindingType::HYBRID_AWARE ||
executorConfig._threadPreferredCoreType == config._threadPreferredCoreType)
return executor;
}
auto newExec = std::make_shared<InferenceEngine::CPUStreamsExecutor>(config);
tbbThreadsCreated = true;
cpuStreamsExecutors.emplace_back(std::make_pair(config, newExec));
return newExec;
}
size_t ExecutorManagerImpl::get_executors_number() const {
std::lock_guard<std::mutex> guard(taskExecutorMutex);
return executors.size();
}
size_t ExecutorManagerImpl::get_idle_cpu_streams_executors_number() const {
std::lock_guard<std::mutex> guard(streamExecutorMutex);
return cpuStreamsExecutors.size();
}
void ExecutorManagerImpl::clear(const std::string& id) {
std::lock_guard<std::mutex> stream_guard(streamExecutorMutex);
std::lock_guard<std::mutex> task_guard(taskExecutorMutex);
if (id.empty()) {
executors.clear();
cpuStreamsExecutors.clear();
} else {
executors.erase(id);
cpuStreamsExecutors.erase(std::remove_if(cpuStreamsExecutors.begin(),
cpuStreamsExecutors.end(),
[&](const std::pair<InferenceEngine::IStreamsExecutor::Config,
InferenceEngine::IStreamsExecutor::Ptr>& it) {
return it.first._name == id;
}),
cpuStreamsExecutors.end());
}
}
namespace {
class ExecutorManagerHolder {
std::mutex _mutex;
std::weak_ptr<ExecutorManager> _manager;
public:
ExecutorManagerHolder(const ExecutorManagerHolder&) = delete;
ExecutorManagerHolder& operator=(const ExecutorManagerHolder&) = delete;
ExecutorManagerHolder() = default;
std::shared_ptr<ov::ExecutorManager> get() {
std::lock_guard<std::mutex> lock(_mutex);
auto manager = _manager.lock();
if (!manager) {
_manager = manager = std::make_shared<ExecutorManagerImpl>();
}
return manager;
}
};
} // namespace
std::shared_ptr<ExecutorManager> executor_manager() {
static ExecutorManagerHolder executorManagerHolder;
return executorManagerHolder.get();
}
} // namespace ov

127
src/inference/src/threading/ie_executor_manager.cpp
View File

@ -5,6 +5,8 @@
#include "threading/ie_executor_manager.hpp"
#include "ie_parallel.hpp"
#include "openvino/runtime/properties.hpp"
#include "openvino/runtime/threading/executor_manager.hpp"
#include "threading/ie_cpu_streams_executor.hpp"
#if IE_THREAD == IE_THREAD_TBB || IE_THREAD == IE_THREAD_TBB_AUTO
# if (TBB_INTERFACE_VERSION < 12000)
@ -23,7 +25,7 @@ namespace InferenceEngine {
namespace {
class ExecutorManagerImpl : public ExecutorManager {
public:
~ExecutorManagerImpl();
ExecutorManagerImpl(const std::shared_ptr<ov::ExecutorManager>& manager);
ITaskExecutor::Ptr getExecutor(const std::string& id) override;
IStreamsExecutor::Ptr getIdleCPUStreamsExecutor(const IStreamsExecutor::Config& config) override;
size_t getExecutorsNumber() const override;
@ -33,134 +35,47 @@ public:
bool getTbbFlag() override;
private:
void resetTbb();
std::unordered_map<std::string, ITaskExecutor::Ptr> executors;
std::vector<std::pair<IStreamsExecutor::Config, IStreamsExecutor::Ptr>> cpuStreamsExecutors;
mutable std::mutex streamExecutorMutex;
mutable std::mutex taskExecutorMutex;
bool tbbTerminateFlag = false;
mutable std::mutex tbbMutex;
bool tbbThreadsCreated = false;
#if IE_THREAD == IE_THREAD_TBB || IE_THREAD == IE_THREAD_TBB_AUTO
# if (TBB_INTERFACE_VERSION < 12000)
std::shared_ptr<tbb::task_scheduler_init> tbbTaskScheduler = nullptr;
# else
std::shared_ptr<oneapi::tbb::task_scheduler_handle> tbbTaskScheduler = nullptr;
# endif
#endif
std::shared_ptr<ov::ExecutorManager> m_manager;
std::shared_ptr<ov::ExecutorManager> get_ov_manager() const override {
return m_manager;
}
};
} // namespace
ExecutorManagerImpl::~ExecutorManagerImpl() {
resetTbb();
}
ExecutorManagerImpl::ExecutorManagerImpl(const std::shared_ptr<ov::ExecutorManager>& manager) : m_manager(manager) {}
void ExecutorManagerImpl::setTbbFlag(bool flag) {
std::lock_guard<std::mutex> guard(tbbMutex);
tbbTerminateFlag = flag;
#if IE_THREAD == IE_THREAD_TBB || IE_THREAD == IE_THREAD_TBB_AUTO
if (tbbTerminateFlag) {
if (!tbbTaskScheduler) {
# if (TBB_INTERFACE_VERSION < 12000)
tbbTaskScheduler = std::make_shared<tbb::task_scheduler_init>();
# elif (TBB_INTERFACE_VERSION < 12060)
tbbTaskScheduler =
std::make_shared<oneapi::tbb::task_scheduler_handle>(oneapi::tbb::task_scheduler_handle::get());
# else
tbbTaskScheduler = std::make_shared<oneapi::tbb::task_scheduler_handle>(tbb::attach{});
# endif
}
} else {
tbbTaskScheduler = nullptr;
}
#endif
m_manager->set_property({{ov::force_tbb_terminate.name(), flag}});
}
bool ExecutorManagerImpl::getTbbFlag() {
std::lock_guard<std::mutex> guard(tbbMutex);
return tbbTerminateFlag;
}
void ExecutorManagerImpl::resetTbb() {
std::lock_guard<std::mutex> guard(tbbMutex);
if (tbbTerminateFlag) {
#if IE_THREAD == IE_THREAD_TBB || IE_THREAD == IE_THREAD_TBB_AUTO
if (tbbTaskScheduler && tbbThreadsCreated) {
# if (TBB_INTERFACE_VERSION < 12000)
tbbTaskScheduler->terminate();
# else
tbb::finalize(*tbbTaskScheduler, std::nothrow);
# endif
}
tbbThreadsCreated = false;
tbbTaskScheduler = nullptr;
#endif
tbbTerminateFlag = false;
}
return m_manager->get_property(ov::force_tbb_terminate.name()).as<bool>();
}
ITaskExecutor::Ptr ExecutorManagerImpl::getExecutor(const std::string& id) {
std::lock_guard<std::mutex> guard(taskExecutorMutex);
auto foundEntry = executors.find(id);
if (foundEntry == executors.end()) {
auto newExec = std::make_shared<CPUStreamsExecutor>(IStreamsExecutor::Config{id});
tbbThreadsCreated = true;
executors[id] = newExec;
return newExec;
}
return foundEntry->second;
return m_manager->get_executor(id);
}
IStreamsExecutor::Ptr ExecutorManagerImpl::getIdleCPUStreamsExecutor(const IStreamsExecutor::Config& config) {
std::lock_guard<std::mutex> guard(streamExecutorMutex);
for (const auto& it : cpuStreamsExecutors) {
const auto& executor = it.second;
if (executor.use_count() != 1)
continue;
const auto& executorConfig = it.first;
if (executorConfig._name == config._name && executorConfig._streams == config._streams &&
executorConfig._threadsPerStream == config._threadsPerStream &&
executorConfig._threadBindingType == config._threadBindingType &&
executorConfig._threadBindingStep == config._threadBindingStep &&
executorConfig._threadBindingOffset == config._threadBindingOffset)
if (executorConfig._threadBindingType != IStreamsExecutor::ThreadBindingType::HYBRID_AWARE ||
executorConfig._threadPreferredCoreType == config._threadPreferredCoreType)
return executor;
}
auto newExec = std::make_shared<CPUStreamsExecutor>(config);
tbbThreadsCreated = true;
cpuStreamsExecutors.emplace_back(std::make_pair(config, newExec));
return newExec;
return m_manager->get_idle_cpu_streams_executor(config);
}
size_t ExecutorManagerImpl::getExecutorsNumber() const {
std::lock_guard<std::mutex> guard(taskExecutorMutex);
return executors.size();
return m_manager->get_executors_number();
}
size_t ExecutorManagerImpl::getIdleCPUStreamsExecutorsNumber() const {
std::lock_guard<std::mutex> guard(streamExecutorMutex);
return cpuStreamsExecutors.size();
return m_manager->get_idle_cpu_streams_executors_number();
}
void ExecutorManagerImpl::clear(const std::string& id) {
std::lock_guard<std::mutex> stream_guard(streamExecutorMutex);
std::lock_guard<std::mutex> task_guard(taskExecutorMutex);
if (id.empty()) {
executors.clear();
cpuStreamsExecutors.clear();
} else {
executors.erase(id);
cpuStreamsExecutors.erase(
std::remove_if(cpuStreamsExecutors.begin(),
cpuStreamsExecutors.end(),
[&](const std::pair<IStreamsExecutor::Config, IStreamsExecutor::Ptr>& it) {
return it.first._name == id;
}),
cpuStreamsExecutors.end());
}
return m_manager->clear(id);
}
std::shared_ptr<InferenceEngine::ExecutorManager> create_old_manager(
const std::shared_ptr<ov::ExecutorManager>& manager) {
return std::make_shared<ExecutorManagerImpl>(manager);
}
namespace {
@ -179,7 +94,7 @@ public:
std::lock_guard<std::mutex> lock(_mutex);
auto manager = _manager.lock();
if (!manager) {
_manager = manager = std::make_shared<ExecutorManagerImpl>();
_manager = manager = create_old_manager(ov::executor_manager());
}
return manager;
}

6
src/plugins/template/src/plugin.cpp
View File

@ -35,7 +35,7 @@ Plugin::Plugin() {
_backend = ngraph::runtime::Backend::create();
// create default stream executor with a given name
_waitExecutor = get_executor_manager()->getIdleCPUStreamsExecutor({wait_executor_name});
_waitExecutor = get_executor_manager()->get_idle_cpu_streams_executor({wait_executor_name});
}
// ! [plugin:ctor]
@ -96,7 +96,7 @@ std::shared_ptr<ov::ICompiledModel> TemplatePlugin::Plugin::compile_model(const
auto compiled_model =
std::make_shared<CompiledModel>(model->clone(),
shared_from_this(),
get_executor_manager()->getIdleCPUStreamsExecutor(streamsExecutorConfig),
get_executor_manager()->get_idle_cpu_streams_executor(streamsExecutorConfig),
fullConfig);
return compiled_model;
}
@ -136,7 +136,7 @@ std::shared_ptr<ov::ICompiledModel> TemplatePlugin::Plugin::import_model(std::is
auto compiled_model =
std::make_shared<CompiledModel>(ov_model,
shared_from_this(),
get_executor_manager()->getIdleCPUStreamsExecutor(streamsExecutorConfig),
get_executor_manager()->get_idle_cpu_streams_executor(streamsExecutorConfig),
fullConfig);
return compiled_model;
}