From ff00817bb7d711a6b13e5dffb9413776127f858e Mon Sep 17 00:00:00 2001 From: Denis Orlov Date: Mon, 8 Jun 2020 18:43:12 +0300 Subject: [PATCH] [GNA] Support changing the execution mode in runtime (#801) --- .../src/gna_plugin/gna_device.cpp | 27 ++++++------ .../src/gna_plugin/gna_device.hpp | 22 ++++------ .../src/gna_plugin/gna_executable_network.hpp | 31 +++++++++++++ .../src/gna_plugin/gna_plugin.cpp | 16 +++---- .../src/gna_plugin/gna_plugin_query_api.cpp | 14 ++---- .../ie_class/ie_class.cpp | 43 ++++++++++++++++--- .../shared_tests/ie_class/ie_class.hpp | 36 ++++++++++++++++ 7 files changed, 137 insertions(+), 52 deletions(-) diff --git a/inference-engine/src/gna_plugin/gna_device.cpp b/inference-engine/src/gna_plugin/gna_device.cpp index aec8699036c..b92214b25c9 100644 --- a/inference-engine/src/gna_plugin/gna_device.cpp +++ b/inference-engine/src/gna_plugin/gna_device.cpp @@ -52,7 +52,8 @@ void GNADeviceHelper::free(void * ptr) { #if GNA_LIB_VER == 1 uint32_t GNADeviceHelper::propagate(const intel_nnet_type_t *pNeuralNetwork, const uint32_t *pActiveIndices, - uint32_t nActiveIndices) { + uint32_t nActiveIndices, + intel_gna_proc_t nGNAProcType) { uint32_t reqId; nGNAStatus = GNAPropagateForward(nGNAHandle, pNeuralNetwork, @@ -65,14 +66,20 @@ void GNADeviceHelper::setUpActiveList(const uint32_t requestConfigId, uint32_t l const auto status = Gna2RequestConfigEnableActiveList(requestConfigId, layerIndex, num_active_indices, ptr_active_indices); checkGna2Status(status); } -void GNADeviceHelper::propagateSync(const uint32_t requestConfigId) { - wait(propagate(requestConfigId)); +void GNADeviceHelper::propagateSync(const uint32_t requestConfigId, Gna2AccelerationMode gna2AccelerationMode) { + wait(propagate(requestConfigId, gna2AccelerationMode)); } -uint32_t GNADeviceHelper::propagate(const uint32_t requestConfigId) { +uint32_t GNADeviceHelper::propagate(const uint32_t requestConfigId, Gna2AccelerationMode gna2AccelerationMode) { uint32_t reqId; - const auto status = Gna2RequestEnqueue(requestConfigId, &reqId); - checkGna2Status(status); + if (gna2AccelerationMode == Gna2AccelerationModeHardware && + detectedGnaDevVersion == Gna2DeviceVersionSoftwareEmulation) { + gnawarn() << "GNA Device not detected, consider using other mode of acceleration"; + } + const auto status1 = Gna2RequestConfigSetAccelerationMode(requestConfigId, gna2AccelerationMode); + checkGna2Status(status1); + const auto status2 = Gna2RequestEnqueue(requestConfigId, &reqId); + checkGna2Status(status2); return reqId; } @@ -84,7 +91,7 @@ uint32_t GNADeviceHelper::createModel(const Gna2Model& gnaModel) const { return modelId; } -void GNADeviceHelper::releseModel(const uint32_t model_id) { +void GNADeviceHelper::releaseModel(const uint32_t model_id) { const auto status = Gna2ModelRelease(model_id); checkGna2Status(status); } @@ -93,8 +100,6 @@ uint32_t GNADeviceHelper::createRequestConfig(const uint32_t model_id) { uint32_t reqConfId; auto status = Gna2RequestConfigCreate(model_id, &reqConfId); checkGna2Status(status); - status = Gna2RequestConfigSetAccelerationMode(reqConfId, gna2AccelerationMode); - checkGna2Status(status); if (gna2HwConsistency != Gna2DeviceVersionSoftwareEmulation) { status = Gna2RequestConfigEnableHardwareConsistency(reqConfId, gna2HwConsistency); checkGna2Status(status); @@ -350,10 +355,6 @@ void GNADeviceHelper::open(uint8_t n_threads) { #else auto status = Gna2DeviceGetVersion(nGnaDeviceIndex, &detectedGnaDevVersion); checkGna2Status(status); - if (gna2AccelerationMode == Gna2AccelerationModeHardware && - detectedGnaDevVersion == Gna2DeviceVersionSoftwareEmulation) { - gnalog() << "GNA Device not detected, consider using other mode of acceleration"; - } status = Gna2DeviceOpen(nGnaDeviceIndex); checkGna2Status(status); // TODO: GNA2: uncomment when scratchpad repaired diff --git a/inference-engine/src/gna_plugin/gna_device.hpp b/inference-engine/src/gna_plugin/gna_device.hpp index f122445907a..99ce1a1b08b 100644 --- a/inference-engine/src/gna_plugin/gna_device.hpp +++ b/inference-engine/src/gna_plugin/gna_device.hpp @@ -34,12 +34,10 @@ class GNADeviceHelper { #if GNA_LIB_VER == 1 intel_gna_status_t nGNAStatus = GNA_NOERROR; intel_gna_handle_t nGNAHandle = 0; - intel_gna_proc_t nGNAProcType = GNA_AUTO; intel_gna_perf_t nGNAPerfResults; intel_gna_perf_t nGNAPerfResultsTotal; #else uint32_t nGnaDeviceIndex = 0; - Gna2AccelerationMode gna2AccelerationMode = Gna2AccelerationModeAuto; Gna2DeviceVersion gna2HwConsistency = Gna2DeviceVersionSoftwareEmulation; Gna2DeviceVersion detectedGnaDevVersion = Gna2DeviceVersionSoftwareEmulation; @@ -59,19 +57,15 @@ class GNADeviceHelper { bool deviceOpened = false; public: #if GNA_LIB_VER == 1 - explicit GNADeviceHelper(intel_gna_proc_t proc_type = GNA_AUTO, - uint8_t lib_async_n_threads = 1, + explicit GNADeviceHelper(uint8_t lib_async_n_threads = 1, bool use_openmp = false, bool isPerformanceMeasuring = false) : - nGNAProcType(proc_type), isPerformanceMeasuring(isPerformanceMeasuring) { #else - explicit GNADeviceHelper(Gna2AccelerationMode gna2accMode = Gna2AccelerationModeAuto, - Gna2DeviceVersion gna2HwConsistency = Gna2DeviceVersionSoftwareEmulation, + explicit GNADeviceHelper(Gna2DeviceVersion gna2HwConsistency = Gna2DeviceVersionSoftwareEmulation, uint8_t lib_async_n_threads = 1, bool use_openmp = false, bool isPerformanceMeasuring = false) : - gna2AccelerationMode(gna2accMode), gna2HwConsistency(gna2HwConsistency), isPerformanceMeasuring(isPerformanceMeasuring) { #endif @@ -97,21 +91,23 @@ public: #if GNA_LIB_VER == 1 void propagateSync(const intel_nnet_type_t *pNeuralNetwork, const uint32_t *pActiveIndices, - uint32_t nActiveIndices); + uint32_t nActiveIndices, + intel_gna_proc_t nGNAProcType); uint32_t propagate(const intel_nnet_type_t *pNeuralNetwork, const uint32_t *pActiveIndices, - uint32_t nActiveIndices); + uint32_t nActiveIndices, + intel_gna_proc_t nGNAProcType); #else void setUpActiveList(unsigned req_config_id, uint32_t layerIndex, uint32_t* ptr_active_indices, uint32_t num_active_indices); - void propagateSync(const uint32_t requestConfigId); - uint32_t propagate(const uint32_t requestConfigId); + void propagateSync(const uint32_t requestConfigId, Gna2AccelerationMode gna2AccelerationMode); + uint32_t propagate(const uint32_t requestConfigId, Gna2AccelerationMode gna2AccelerationMode); #if GNA_LIB_VER == 2 uint32_t createModel(const Gna2Model& gnaModel) const; #else uint32_t createModel(const intel_nnet_type_t& intel_nnet_type); #endif - void releseModel(const uint32_t model_id); + void releaseModel(const uint32_t model_id); uint32_t createRequestConfig(const uint32_t model_id); bool hasGnaHw() const { return Gna2DeviceVersionSoftwareEmulation != detectedGnaDevVersion; diff --git a/inference-engine/src/gna_plugin/gna_executable_network.hpp b/inference-engine/src/gna_plugin/gna_executable_network.hpp index 90f01ff17cb..f259da2a81c 100644 --- a/inference-engine/src/gna_plugin/gna_executable_network.hpp +++ b/inference-engine/src/gna_plugin/gna_executable_network.hpp @@ -12,6 +12,7 @@ #include #include "gna_infer_request.hpp" #include "gna_plugin.hpp" +#include #include #include @@ -67,6 +68,36 @@ class GNAExecutableNetwork : public InferenceEngine::ExecutableNetworkThreadSafe THROW_IE_EXCEPTION << NOT_IMPLEMENTED_str; } + void SetConfig(const std::map& config, + InferenceEngine::ResponseDesc* /* resp */) override { + using namespace InferenceEngine::GNAConfigParams; + if (config.empty()) { + THROW_IE_EXCEPTION << "The list of configuration values is empty"; + } + for (auto&& item : config) { + if (item.first != KEY_GNA_DEVICE_MODE) { + THROW_IE_EXCEPTION << "The following config value cannot be changed dynamically for ExecutableNetwork in the GNA plugin: " + << item.first << ". Only " << KEY_GNA_DEVICE_MODE << " is supported."; + } + } + + InferenceEngine::Parameter old_mode_parameter; + GetConfig(KEY_GNA_DEVICE_MODE, old_mode_parameter, {}); + auto old_mode = old_mode_parameter.as(); + if (old_mode == InferenceEngine::GNAConfigParams::GNA_SW_FP32) { + THROW_IE_EXCEPTION << "Dynamic switching from GNA_SW_FP32 mode is not supported for ExecutableNetwork."; + } + + auto new_mode = config.begin()->second.as(); + if (new_mode == InferenceEngine::GNAConfigParams::GNA_SW_FP32) { + THROW_IE_EXCEPTION << "Dynamic switching to GNA_SW_FP32 mode is not supported for ExecutableNetwork."; + } + + std::map configForPlugin; + configForPlugin[KEY_GNA_DEVICE_MODE] = new_mode; + plg->SetConfig(configForPlugin); + } + void GetConfig(const std::string &name, InferenceEngine::Parameter &result, InferenceEngine::ResponseDesc* /*resp*/) const override { diff --git a/inference-engine/src/gna_plugin/gna_plugin.cpp b/inference-engine/src/gna_plugin/gna_plugin.cpp index b784ccd9305..0e190522ce0 100644 --- a/inference-engine/src/gna_plugin/gna_plugin.cpp +++ b/inference-engine/src/gna_plugin/gna_plugin.cpp @@ -323,13 +323,11 @@ void GNAPlugin::Init() { void GNAPlugin::InitGNADevice() { #if GNA_LIB_VER == 1 - gnadevice = std::make_shared(config.gna_proc_type, - gnaFlags->gna_lib_async_threads_num, - gnaFlags->gna_openmp_multithreading, - gnaFlags->performance_counting); + gnadevice = std::make_shared(gnaFlags->gna_lib_async_threads_num, + gnaFlags->gna_openmp_multithreading, + gnaFlags->performance_counting); #else - gnadevice = std::make_shared(config.pluginGna2AccMode, - config.pluginGna2DeviceConsistent, + gnadevice = std::make_shared(config.pluginGna2DeviceConsistent, gnaFlags->gna_lib_async_threads_num, gnaFlags->gna_openmp_multithreading, gnaFlags->performance_counting); @@ -811,7 +809,7 @@ void GNAPlugin::DumpXNNToFile() const { gnadevice->dumpXnnForDeviceVersion(modelId, dumpStream, *reinterpret_cast(&versionInt)); } - gnadevice->releseModel(modelId); + gnadevice->releaseModel(modelId); #endif } @@ -934,12 +932,12 @@ uint32_t GNAPlugin::QueueInference(const InferenceEngine::BlobMap &inputs, Infer } else { #if GNA_LIB_VER == 1 auto nnet = std::get<0>(*freeNnet).get(); - std::get<1>(*freeNnet) = gnadevice->propagate(&nnet->obj, ptr_active_indices, num_active_indices); + std::get<1>(*freeNnet) = gnadevice->propagate(&nnet->obj, ptr_active_indices, num_active_indices, config.gna_proc_type); #else const auto reqConfigId = std::get<0>(*freeNnet); if (ptr_active_indices != nullptr && num_active_indices > 0 && activeLayerIndex != 0xffffffff) gnadevice->setUpActiveList(reqConfigId, activeLayerIndex, ptr_active_indices, num_active_indices); - std::get<1>(*freeNnet) = gnadevice->propagate(reqConfigId); + std::get<1>(*freeNnet) = gnadevice->propagate(reqConfigId, config.pluginGna2AccMode); #endif } diff --git a/inference-engine/src/gna_plugin/gna_plugin_query_api.cpp b/inference-engine/src/gna_plugin/gna_plugin_query_api.cpp index ec7aad5b036..c078675f1fa 100644 --- a/inference-engine/src/gna_plugin/gna_plugin_query_api.cpp +++ b/inference-engine/src/gna_plugin/gna_plugin_query_api.cpp @@ -67,24 +67,16 @@ Parameter GNAPlugin::GetAvailableDevices() const { std::vector devices; // probing for gna-sw-exact, or gna-sw implementation part of libgna try { -#if GNA_LIB_VER == 2 - GNADeviceHelper swHelper(Gna2AccelerationModeSoftware); -#else - GNADeviceHelper swHelper(GNA_SOFTWARE); -#endif + GNADeviceHelper swHelper; devices.push_back("GNA_SW"); }catch(...) {} try { -#if GNA_LIB_VER == 2 - GNADeviceHelper hwHelper(Gna2AccelerationModeHardware); -#else - GNADeviceHelper hwHelper(GNA_HARDWARE); -#endif + GNADeviceHelper hwHelper; #if GNA_LIB_VER == 1 try { intel_nnet_type_t neuralNetwork = { 0 }; - hwHelper.propagate(&neuralNetwork, nullptr, 0); + hwHelper.propagate(&neuralNetwork, nullptr, 0, GNA_HARDWARE); }catch (...) { if (hwHelper.getGNAStatus() != GNA_DEVNOTFOUND) { devices.push_back("GNA_HW"); diff --git a/inference-engine/tests_deprecated/functional/gna/shared_tests_instance/ie_class/ie_class.cpp b/inference-engine/tests_deprecated/functional/gna/shared_tests_instance/ie_class/ie_class.cpp index a83653b43d9..97e70556911 100644 --- a/inference-engine/tests_deprecated/functional/gna/shared_tests_instance/ie_class/ie_class.cpp +++ b/inference-engine/tests_deprecated/functional/gna/shared_tests_instance/ie_class/ie_class.cpp @@ -3,12 +3,7 @@ // #include "ie_class.hpp" - -// Copyright (C) 2018-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "ie_class.hpp" +#include // // IE Class Common tests with @@ -110,6 +105,42 @@ INSTANTIATE_TEST_CASE_P( IEClassExecutableNetworkSetConfigTest, IEClassExecutableNetworkSetConfigTest, ::testing::Values("GNA")); +INSTANTIATE_TEST_CASE_P( + IEClassExecutableNetworkSupportedConfigTest, IEClassExecutableNetworkSupportedConfigTest, + ::testing::Combine(::testing::Values("GNA"), + ::testing::Values(std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_HW), + std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW), + std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW_EXACT), + std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_AUTO)))); + +INSTANTIATE_TEST_CASE_P( + IEClassExecutableNetworkUnsupportedConfigTest, IEClassExecutableNetworkUnsupportedConfigTest, + ::testing::Combine(::testing::Values("GNA"), + ::testing::Values(std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW_FP32), + std::make_pair(GNA_CONFIG_KEY(SCALE_FACTOR), "5"), + std::make_pair(CONFIG_KEY(EXCLUSIVE_ASYNC_REQUESTS), CONFIG_VALUE(YES)), + std::make_pair(GNA_CONFIG_KEY(COMPACT_MODE), CONFIG_VALUE(NO))))); + +using IEClassExecutableNetworkSetConfigFromFp32Test = IEClassExecutableNetworkGetMetricTestForSpecificConfig; +TEST_P(IEClassExecutableNetworkSetConfigFromFp32Test, SetConfigFromFp32Throws) { + Core ie; + + std::map initialConfig; + initialConfig[GNA_CONFIG_KEY(DEVICE_MODE)] = GNAConfigParams::GNA_SW_FP32; + ExecutableNetwork exeNetwork = ie.LoadNetwork(simpleNetwork, deviceName, initialConfig); + + ASSERT_THROW(exeNetwork.SetConfig({ { configKey, configValue } }), InferenceEngineException); +} + +INSTANTIATE_TEST_CASE_P( + IEClassExecutableNetworkSetConfigFromFp32Test, IEClassExecutableNetworkSetConfigFromFp32Test, + ::testing::Combine(::testing::Values("GNA"), + ::testing::Values(std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_HW), + std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW), + std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW_EXACT), + std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW_FP32), + std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_AUTO)))); + // IE Class Query network INSTANTIATE_TEST_CASE_P( diff --git a/inference-engine/tests_deprecated/functional/shared_tests/ie_class/ie_class.hpp b/inference-engine/tests_deprecated/functional/shared_tests/ie_class/ie_class.hpp index 8191938c382..52fd8e25def 100644 --- a/inference-engine/tests_deprecated/functional/shared_tests/ie_class/ie_class.hpp +++ b/inference-engine/tests_deprecated/functional/shared_tests/ie_class/ie_class.hpp @@ -785,6 +785,21 @@ public: } }; +class IEClassExecutableNetworkGetMetricTestForSpecificConfig : public IEClassNetworkTest, +public WithParamInterface>> { +protected: + std::string deviceName; + std::string configKey; + std::string configValue; +public: + virtual void SetUp() { + IEClassNetworkTest::SetUp(); + deviceName = get<0>(GetParam()); + configKey = get<1>(GetParam()).first; + configValue = get<1>(GetParam()).second; + } +}; + #define ASSERT_EXEC_METRIC_SUPPORTED(metricName) \ { \ std::vector metrics = \ @@ -907,6 +922,27 @@ TEST_P(IEClassExecutableNetworkSetConfigTest, SetConfigThrows) { ASSERT_THROW(exeNetwork.SetConfig({ { "unsupported_config", "some_value" } }), InferenceEngineException); } +using IEClassExecutableNetworkSupportedConfigTest = IEClassExecutableNetworkGetMetricTestForSpecificConfig; +TEST_P(IEClassExecutableNetworkSupportedConfigTest, SupportedConfigWorks) { + Core ie; + Parameter p; + + ExecutableNetwork exeNetwork = ie.LoadNetwork(simpleNetwork, deviceName); + + ASSERT_NO_THROW(exeNetwork.SetConfig({ { configKey, configValue } })); + ASSERT_NO_THROW(p = exeNetwork.GetConfig( configKey )); + ASSERT_EQ(p, configValue); +} + +using IEClassExecutableNetworkUnsupportedConfigTest = IEClassExecutableNetworkGetMetricTestForSpecificConfig; +TEST_P(IEClassExecutableNetworkUnsupportedConfigTest, UnsupportedConfigThrows) { + Core ie; + + ExecutableNetwork exeNetwork = ie.LoadNetwork(simpleNetwork, deviceName); + + ASSERT_THROW(exeNetwork.SetConfig({ { configKey, configValue } }), InferenceEngineException); +} + using IEClassExecutableNetworkGetConfigTest = IEClassExecutableNetworkGetMetricTest; TEST_P(IEClassExecutableNetworkGetConfigTest, GetConfigNoEmptyNoThrow) { Core ie;