IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[GNA] Support changing the execution mode in runtime (#801)

Browse Source
This commit is contained in:
Denis Orlov 2020-06-08 18:43:12 +03:00 committed by GitHub
parent eefaf56075
commit ff00817bb7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 137 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
inference-engine/src/gna_plugin/gna_device.cpp
View File

@ -52,7 +52,8 @@ void GNADeviceHelper::free(void * ptr) {
#if GNA_LIB_VER == 1
uint32_t GNADeviceHelper::propagate(const intel_nnet_type_t *pNeuralNetwork,
const uint32_t *pActiveIndices,
uint32_t nActiveIndices) {
uint32_t nActiveIndices,
intel_gna_proc_t nGNAProcType) {
uint32_t reqId;
nGNAStatus = GNAPropagateForward(nGNAHandle, pNeuralNetwork,
@ -65,14 +66,20 @@ void GNADeviceHelper::setUpActiveList(const uint32_t requestConfigId, uint32_t l
const auto status = Gna2RequestConfigEnableActiveList(requestConfigId, layerIndex, num_active_indices, ptr_active_indices);
checkGna2Status(status);
}
void GNADeviceHelper::propagateSync(const uint32_t requestConfigId) {
wait(propagate(requestConfigId));
void GNADeviceHelper::propagateSync(const uint32_t requestConfigId, Gna2AccelerationMode gna2AccelerationMode) {
wait(propagate(requestConfigId, gna2AccelerationMode));
}
uint32_t GNADeviceHelper::propagate(const uint32_t requestConfigId) {
uint32_t GNADeviceHelper::propagate(const uint32_t requestConfigId, Gna2AccelerationMode gna2AccelerationMode) {
uint32_t reqId;
const auto status = Gna2RequestEnqueue(requestConfigId, &reqId);
checkGna2Status(status);
if (gna2AccelerationMode == Gna2AccelerationModeHardware &&
detectedGnaDevVersion == Gna2DeviceVersionSoftwareEmulation) {
gnawarn() << "GNA Device not detected, consider using other mode of acceleration";
}
const auto status1 = Gna2RequestConfigSetAccelerationMode(requestConfigId, gna2AccelerationMode);
checkGna2Status(status1);
const auto status2 = Gna2RequestEnqueue(requestConfigId, &reqId);
checkGna2Status(status2);
return reqId;
}
@ -84,7 +91,7 @@ uint32_t GNADeviceHelper::createModel(const Gna2Model& gnaModel) const {
return modelId;
}
void GNADeviceHelper::releseModel(const uint32_t model_id) {
void GNADeviceHelper::releaseModel(const uint32_t model_id) {
const auto status = Gna2ModelRelease(model_id);
checkGna2Status(status);
}
@ -93,8 +100,6 @@ uint32_t GNADeviceHelper::createRequestConfig(const uint32_t model_id) {
uint32_t reqConfId;
auto status = Gna2RequestConfigCreate(model_id, &reqConfId);
checkGna2Status(status);
status = Gna2RequestConfigSetAccelerationMode(reqConfId, gna2AccelerationMode);
checkGna2Status(status);
if (gna2HwConsistency != Gna2DeviceVersionSoftwareEmulation) {
status = Gna2RequestConfigEnableHardwareConsistency(reqConfId, gna2HwConsistency);
checkGna2Status(status);
@ -350,10 +355,6 @@ void GNADeviceHelper::open(uint8_t n_threads) {
#else
auto status = Gna2DeviceGetVersion(nGnaDeviceIndex, &detectedGnaDevVersion);
checkGna2Status(status);
if (gna2AccelerationMode == Gna2AccelerationModeHardware &&
detectedGnaDevVersion == Gna2DeviceVersionSoftwareEmulation) {
gnalog() << "GNA Device not detected, consider using other mode of acceleration";
}
status = Gna2DeviceOpen(nGnaDeviceIndex);
checkGna2Status(status);
// TODO: GNA2: uncomment when scratchpad repaired

22
inference-engine/src/gna_plugin/gna_device.hpp
View File

@ -34,12 +34,10 @@ class GNADeviceHelper {
#if GNA_LIB_VER == 1
intel_gna_status_t nGNAStatus = GNA_NOERROR;
intel_gna_handle_t nGNAHandle = 0;
intel_gna_proc_t nGNAProcType = GNA_AUTO;
intel_gna_perf_t nGNAPerfResults;
intel_gna_perf_t nGNAPerfResultsTotal;
#else
uint32_t nGnaDeviceIndex = 0;
Gna2AccelerationMode gna2AccelerationMode = Gna2AccelerationModeAuto;
Gna2DeviceVersion gna2HwConsistency = Gna2DeviceVersionSoftwareEmulation;
Gna2DeviceVersion detectedGnaDevVersion = Gna2DeviceVersionSoftwareEmulation;
@ -59,19 +57,15 @@ class GNADeviceHelper {
bool deviceOpened = false;
public:
#if GNA_LIB_VER == 1
explicit GNADeviceHelper(intel_gna_proc_t proc_type = GNA_AUTO,
uint8_t lib_async_n_threads = 1,
explicit GNADeviceHelper(uint8_t lib_async_n_threads = 1,
bool use_openmp = false,
bool isPerformanceMeasuring = false) :
nGNAProcType(proc_type),
isPerformanceMeasuring(isPerformanceMeasuring) {
#else
explicit GNADeviceHelper(Gna2AccelerationMode gna2accMode = Gna2AccelerationModeAuto,
Gna2DeviceVersion gna2HwConsistency = Gna2DeviceVersionSoftwareEmulation,
explicit GNADeviceHelper(Gna2DeviceVersion gna2HwConsistency = Gna2DeviceVersionSoftwareEmulation,
uint8_t lib_async_n_threads = 1,
bool use_openmp = false,
bool isPerformanceMeasuring = false) :
gna2AccelerationMode(gna2accMode),
gna2HwConsistency(gna2HwConsistency),
isPerformanceMeasuring(isPerformanceMeasuring) {
#endif
@ -97,21 +91,23 @@ public:
#if GNA_LIB_VER == 1
void propagateSync(const intel_nnet_type_t *pNeuralNetwork,
const uint32_t *pActiveIndices,
uint32_t nActiveIndices);
uint32_t nActiveIndices,
intel_gna_proc_t nGNAProcType);
uint32_t propagate(const intel_nnet_type_t *pNeuralNetwork,
const uint32_t *pActiveIndices,
uint32_t nActiveIndices);
uint32_t nActiveIndices,
intel_gna_proc_t nGNAProcType);
#else
void setUpActiveList(unsigned req_config_id, uint32_t layerIndex, uint32_t* ptr_active_indices, uint32_t num_active_indices);
void propagateSync(const uint32_t requestConfigId);
uint32_t propagate(const uint32_t requestConfigId);
void propagateSync(const uint32_t requestConfigId, Gna2AccelerationMode gna2AccelerationMode);
uint32_t propagate(const uint32_t requestConfigId, Gna2AccelerationMode gna2AccelerationMode);
#if GNA_LIB_VER == 2
uint32_t createModel(const Gna2Model& gnaModel) const;
#else
uint32_t createModel(const intel_nnet_type_t& intel_nnet_type);
#endif
void releseModel(const uint32_t model_id);
void releaseModel(const uint32_t model_id);
uint32_t createRequestConfig(const uint32_t model_id);
bool hasGnaHw() const {
return Gna2DeviceVersionSoftwareEmulation != detectedGnaDevVersion;

31
inference-engine/src/gna_plugin/gna_executable_network.hpp
View File

@ -12,6 +12,7 @@
#include <cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp>
#include "gna_infer_request.hpp"
#include "gna_plugin.hpp"
#include <gna/gna_config.hpp>
#include <threading/ie_executor_manager.hpp>
#include <cpp_interfaces/impl/ie_executable_network_thread_safe_async_only.hpp>
@ -67,6 +68,36 @@ class GNAExecutableNetwork : public InferenceEngine::ExecutableNetworkThreadSafe
THROW_IE_EXCEPTION << NOT_IMPLEMENTED_str;
}
void SetConfig(const std::map<std::string, InferenceEngine::Parameter>& config,
InferenceEngine::ResponseDesc* /* resp */) override {
using namespace InferenceEngine::GNAConfigParams;
if (config.empty()) {
THROW_IE_EXCEPTION << "The list of configuration values is empty";
}
for (auto&& item : config) {
if (item.first != KEY_GNA_DEVICE_MODE) {
THROW_IE_EXCEPTION << "The following config value cannot be changed dynamically for ExecutableNetwork in the GNA plugin: "
<< item.first << ". Only " << KEY_GNA_DEVICE_MODE << " is supported.";
}
}
InferenceEngine::Parameter old_mode_parameter;
GetConfig(KEY_GNA_DEVICE_MODE, old_mode_parameter, {});
auto old_mode = old_mode_parameter.as<std::string>();
if (old_mode == InferenceEngine::GNAConfigParams::GNA_SW_FP32) {
THROW_IE_EXCEPTION << "Dynamic switching from GNA_SW_FP32 mode is not supported for ExecutableNetwork.";
}
auto new_mode = config.begin()->second.as<std::string>();
if (new_mode == InferenceEngine::GNAConfigParams::GNA_SW_FP32) {
THROW_IE_EXCEPTION << "Dynamic switching to GNA_SW_FP32 mode is not supported for ExecutableNetwork.";
}
std::map<std::string, std::string> configForPlugin;
configForPlugin[KEY_GNA_DEVICE_MODE] = new_mode;
plg->SetConfig(configForPlugin);
}
void GetConfig(const std::string &name,
InferenceEngine::Parameter &result,
InferenceEngine::ResponseDesc* /*resp*/) const override {

16
inference-engine/src/gna_plugin/gna_plugin.cpp
View File

@ -323,13 +323,11 @@ void GNAPlugin::Init() {
void GNAPlugin::InitGNADevice() {
#if GNA_LIB_VER == 1
gnadevice = std::make_shared<GNADeviceHelper>(config.gna_proc_type,
gnaFlags->gna_lib_async_threads_num,
gnaFlags->gna_openmp_multithreading,
gnaFlags->performance_counting);
gnadevice = std::make_shared<GNADeviceHelper>(gnaFlags->gna_lib_async_threads_num,
gnaFlags->gna_openmp_multithreading,
gnaFlags->performance_counting);
#else
gnadevice = std::make_shared<GNADeviceHelper>(config.pluginGna2AccMode,
config.pluginGna2DeviceConsistent,
gnadevice = std::make_shared<GNADeviceHelper>(config.pluginGna2DeviceConsistent,
gnaFlags->gna_lib_async_threads_num,
gnaFlags->gna_openmp_multithreading,
gnaFlags->performance_counting);
@ -811,7 +809,7 @@ void GNAPlugin::DumpXNNToFile() const {
gnadevice->dumpXnnForDeviceVersion(modelId, dumpStream,
*reinterpret_cast<const Gna2DeviceVersion*>(&versionInt));
}
gnadevice->releseModel(modelId);
gnadevice->releaseModel(modelId);
#endif
}
@ -934,12 +932,12 @@ uint32_t GNAPlugin::QueueInference(const InferenceEngine::BlobMap &inputs, Infer
} else {
#if GNA_LIB_VER == 1
auto nnet = std::get<0>(*freeNnet).get();
std::get<1>(*freeNnet) = gnadevice->propagate(&nnet->obj, ptr_active_indices, num_active_indices);
std::get<1>(*freeNnet) = gnadevice->propagate(&nnet->obj, ptr_active_indices, num_active_indices, config.gna_proc_type);
#else
const auto reqConfigId = std::get<0>(*freeNnet);
if (ptr_active_indices != nullptr && num_active_indices > 0 && activeLayerIndex != 0xffffffff)
gnadevice->setUpActiveList(reqConfigId, activeLayerIndex, ptr_active_indices, num_active_indices);
std::get<1>(*freeNnet) = gnadevice->propagate(reqConfigId);
std::get<1>(*freeNnet) = gnadevice->propagate(reqConfigId, config.pluginGna2AccMode);
#endif
}

14
inference-engine/src/gna_plugin/gna_plugin_query_api.cpp
View File

@ -67,24 +67,16 @@ Parameter GNAPlugin::GetAvailableDevices() const {
std::vector<std::string> devices;
// probing for gna-sw-exact, or gna-sw implementation part of libgna
try {
#if GNA_LIB_VER == 2
GNADeviceHelper swHelper(Gna2AccelerationModeSoftware);
#else
GNADeviceHelper swHelper(GNA_SOFTWARE);
#endif
GNADeviceHelper swHelper;
devices.push_back("GNA_SW");
}catch(...) {}
try {
#if GNA_LIB_VER == 2
GNADeviceHelper hwHelper(Gna2AccelerationModeHardware);
#else
GNADeviceHelper hwHelper(GNA_HARDWARE);
#endif
GNADeviceHelper hwHelper;
#if GNA_LIB_VER == 1
try {
intel_nnet_type_t neuralNetwork = { 0 };
hwHelper.propagate(&neuralNetwork, nullptr, 0);
hwHelper.propagate(&neuralNetwork, nullptr, 0, GNA_HARDWARE);
}catch (...) {
if (hwHelper.getGNAStatus() != GNA_DEVNOTFOUND) {
devices.push_back("GNA_HW");

43
inference-engine/tests_deprecated/functional/gna/shared_tests_instance/ie_class/ie_class.cpp
View File

@ -3,12 +3,7 @@
//
#include "ie_class.hpp"
// Copyright (C) 2018-2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "ie_class.hpp"
#include <gna/gna_config.hpp>
//
// IE Class Common tests with <pluginName, deviceName params>
@ -110,6 +105,42 @@ INSTANTIATE_TEST_CASE_P(
IEClassExecutableNetworkSetConfigTest, IEClassExecutableNetworkSetConfigTest,
::testing::Values("GNA"));
INSTANTIATE_TEST_CASE_P(
IEClassExecutableNetworkSupportedConfigTest, IEClassExecutableNetworkSupportedConfigTest,
::testing::Combine(::testing::Values("GNA"),
::testing::Values(std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_HW),
std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW),
std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW_EXACT),
std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_AUTO))));
INSTANTIATE_TEST_CASE_P(
IEClassExecutableNetworkUnsupportedConfigTest, IEClassExecutableNetworkUnsupportedConfigTest,
::testing::Combine(::testing::Values("GNA"),
::testing::Values(std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW_FP32),
std::make_pair(GNA_CONFIG_KEY(SCALE_FACTOR), "5"),
std::make_pair(CONFIG_KEY(EXCLUSIVE_ASYNC_REQUESTS), CONFIG_VALUE(YES)),
std::make_pair(GNA_CONFIG_KEY(COMPACT_MODE), CONFIG_VALUE(NO)))));
using IEClassExecutableNetworkSetConfigFromFp32Test = IEClassExecutableNetworkGetMetricTestForSpecificConfig;
TEST_P(IEClassExecutableNetworkSetConfigFromFp32Test, SetConfigFromFp32Throws) {
Core ie;
std::map<std::string, std::string> initialConfig;
initialConfig[GNA_CONFIG_KEY(DEVICE_MODE)] = GNAConfigParams::GNA_SW_FP32;
ExecutableNetwork exeNetwork = ie.LoadNetwork(simpleNetwork, deviceName, initialConfig);
ASSERT_THROW(exeNetwork.SetConfig({ { configKey, configValue } }), InferenceEngineException);
}
INSTANTIATE_TEST_CASE_P(
IEClassExecutableNetworkSetConfigFromFp32Test, IEClassExecutableNetworkSetConfigFromFp32Test,
::testing::Combine(::testing::Values("GNA"),
::testing::Values(std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_HW),
std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW),
std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW_EXACT),
std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW_FP32),
std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_AUTO))));
// IE Class Query network
INSTANTIATE_TEST_CASE_P(

36
inference-engine/tests_deprecated/functional/shared_tests/ie_class/ie_class.hpp
View File

@ -785,6 +785,21 @@ public:
}
};
class IEClassExecutableNetworkGetMetricTestForSpecificConfig : public IEClassNetworkTest,
public WithParamInterface<std::tuple<std::string, std::pair<std::string, std::string>>> {
protected:
std::string deviceName;
std::string configKey;
std::string configValue;
public:
virtual void SetUp() {
IEClassNetworkTest::SetUp();
deviceName = get<0>(GetParam());
configKey = get<1>(GetParam()).first;
configValue = get<1>(GetParam()).second;
}
};
#define ASSERT_EXEC_METRIC_SUPPORTED(metricName) \
{ \
std::vector<std::string> metrics = \
@ -907,6 +922,27 @@ TEST_P(IEClassExecutableNetworkSetConfigTest, SetConfigThrows) {
ASSERT_THROW(exeNetwork.SetConfig({ { "unsupported_config", "some_value" } }), InferenceEngineException);
}
using IEClassExecutableNetworkSupportedConfigTest = IEClassExecutableNetworkGetMetricTestForSpecificConfig;
TEST_P(IEClassExecutableNetworkSupportedConfigTest, SupportedConfigWorks) {
Core ie;
Parameter p;
ExecutableNetwork exeNetwork = ie.LoadNetwork(simpleNetwork, deviceName);
ASSERT_NO_THROW(exeNetwork.SetConfig({ { configKey, configValue } }));
ASSERT_NO_THROW(p = exeNetwork.GetConfig( configKey ));
ASSERT_EQ(p, configValue);
}
using IEClassExecutableNetworkUnsupportedConfigTest = IEClassExecutableNetworkGetMetricTestForSpecificConfig;
TEST_P(IEClassExecutableNetworkUnsupportedConfigTest, UnsupportedConfigThrows) {
Core ie;
ExecutableNetwork exeNetwork = ie.LoadNetwork(simpleNetwork, deviceName);
ASSERT_THROW(exeNetwork.SetConfig({ { configKey, configValue } }), InferenceEngineException);
}
using IEClassExecutableNetworkGetConfigTest = IEClassExecutableNetworkGetMetricTest;
TEST_P(IEClassExecutableNetworkGetConfigTest, GetConfigNoEmptyNoThrow) {
Core ie;