[GNA] Support changing the execution mode in runtime (#801)

inference-engine/src/gna_plugin/gna_device.cpp

@@ -52,7 +52,8 @@ void GNADeviceHelper::free(void * ptr) {
 #if GNA_LIB_VER == 1
 uint32_t GNADeviceHelper::propagate(const intel_nnet_type_t *pNeuralNetwork,
                    const uint32_t *pActiveIndices,
-                   uint32_t nActiveIndices) {
+                   uint32_t nActiveIndices,
+                   intel_gna_proc_t nGNAProcType) {
     uint32_t reqId;
 
     nGNAStatus = GNAPropagateForward(nGNAHandle, pNeuralNetwork,
@@ -65,14 +66,20 @@ void GNADeviceHelper::setUpActiveList(const uint32_t requestConfigId, uint32_t l
     const auto status = Gna2RequestConfigEnableActiveList(requestConfigId, layerIndex, num_active_indices, ptr_active_indices);
     checkGna2Status(status);
 }
-void GNADeviceHelper::propagateSync(const uint32_t requestConfigId) {
+void GNADeviceHelper::propagateSync(const uint32_t requestConfigId, Gna2AccelerationMode gna2AccelerationMode) {
-    wait(propagate(requestConfigId));
+    wait(propagate(requestConfigId, gna2AccelerationMode));
 }
 
-uint32_t GNADeviceHelper::propagate(const uint32_t requestConfigId) {
+uint32_t GNADeviceHelper::propagate(const uint32_t requestConfigId, Gna2AccelerationMode gna2AccelerationMode) {
     uint32_t reqId;
-    const auto status = Gna2RequestEnqueue(requestConfigId, &reqId);
+    if (gna2AccelerationMode == Gna2AccelerationModeHardware &&
-    checkGna2Status(status);
+        detectedGnaDevVersion == Gna2DeviceVersionSoftwareEmulation) {
+        gnawarn() << "GNA Device not detected, consider using other mode of acceleration";
+    }
+    const auto status1 = Gna2RequestConfigSetAccelerationMode(requestConfigId, gna2AccelerationMode);
+    checkGna2Status(status1);
+    const auto status2 = Gna2RequestEnqueue(requestConfigId, &reqId);
+    checkGna2Status(status2);
     return reqId;
 }
 
@@ -84,7 +91,7 @@ uint32_t GNADeviceHelper::createModel(const Gna2Model& gnaModel) const {
     return modelId;
 }
 
-void GNADeviceHelper::releseModel(const uint32_t model_id) {
+void GNADeviceHelper::releaseModel(const uint32_t model_id) {
     const auto status = Gna2ModelRelease(model_id);
     checkGna2Status(status);
 }
@@ -93,8 +100,6 @@ uint32_t GNADeviceHelper::createRequestConfig(const uint32_t model_id) {
     uint32_t reqConfId;
     auto status = Gna2RequestConfigCreate(model_id, &reqConfId);
     checkGna2Status(status);
-    status = Gna2RequestConfigSetAccelerationMode(reqConfId, gna2AccelerationMode);
-    checkGna2Status(status);
     if (gna2HwConsistency != Gna2DeviceVersionSoftwareEmulation) {
         status = Gna2RequestConfigEnableHardwareConsistency(reqConfId, gna2HwConsistency);
         checkGna2Status(status);
@@ -350,10 +355,6 @@ void GNADeviceHelper::open(uint8_t n_threads) {
 #else
     auto status = Gna2DeviceGetVersion(nGnaDeviceIndex, &detectedGnaDevVersion);
     checkGna2Status(status);
-    if (gna2AccelerationMode == Gna2AccelerationModeHardware &&
-        detectedGnaDevVersion == Gna2DeviceVersionSoftwareEmulation) {
-        gnalog() << "GNA Device not detected, consider using other mode of acceleration";
-    }
     status = Gna2DeviceOpen(nGnaDeviceIndex);
     checkGna2Status(status);
     // TODO: GNA2: uncomment when scratchpad repaired

inference-engine/src/gna_plugin/gna_device.hpp

@@ -34,12 +34,10 @@ class GNADeviceHelper {
 #if GNA_LIB_VER == 1
     intel_gna_status_t nGNAStatus = GNA_NOERROR;
     intel_gna_handle_t nGNAHandle = 0;
-    intel_gna_proc_t nGNAProcType = GNA_AUTO;
     intel_gna_perf_t nGNAPerfResults;
     intel_gna_perf_t nGNAPerfResultsTotal;
 #else
     uint32_t nGnaDeviceIndex = 0;
-    Gna2AccelerationMode gna2AccelerationMode = Gna2AccelerationModeAuto;
     Gna2DeviceVersion gna2HwConsistency = Gna2DeviceVersionSoftwareEmulation;
     Gna2DeviceVersion detectedGnaDevVersion = Gna2DeviceVersionSoftwareEmulation;
 
@@ -59,19 +57,15 @@ class GNADeviceHelper {
     bool deviceOpened = false;
 public:
 #if GNA_LIB_VER == 1
-    explicit GNADeviceHelper(intel_gna_proc_t proc_type = GNA_AUTO,
+    explicit GNADeviceHelper(uint8_t lib_async_n_threads = 1,
-                            uint8_t lib_async_n_threads = 1,
                             bool use_openmp = false,
                             bool isPerformanceMeasuring = false) :
-                                    nGNAProcType(proc_type),
                                     isPerformanceMeasuring(isPerformanceMeasuring) {
 #else
-     explicit GNADeviceHelper(Gna2AccelerationMode gna2accMode = Gna2AccelerationModeAuto,
+     explicit GNADeviceHelper(Gna2DeviceVersion gna2HwConsistency = Gna2DeviceVersionSoftwareEmulation,
-         Gna2DeviceVersion gna2HwConsistency = Gna2DeviceVersionSoftwareEmulation,
          uint8_t lib_async_n_threads = 1,
          bool use_openmp = false,
          bool isPerformanceMeasuring = false) :
-         gna2AccelerationMode(gna2accMode),
          gna2HwConsistency(gna2HwConsistency),
          isPerformanceMeasuring(isPerformanceMeasuring) {
 #endif
@@ -97,21 +91,23 @@ public:
 #if GNA_LIB_VER == 1
     void propagateSync(const intel_nnet_type_t *pNeuralNetwork,
                        const uint32_t *pActiveIndices,
-                       uint32_t nActiveIndices);
+                       uint32_t nActiveIndices,
+                       intel_gna_proc_t nGNAProcType);
 
     uint32_t propagate(const intel_nnet_type_t *pNeuralNetwork,
                        const uint32_t *pActiveIndices,
-                       uint32_t nActiveIndices);
+                       uint32_t nActiveIndices,
+                       intel_gna_proc_t nGNAProcType);
 #else
     void setUpActiveList(unsigned req_config_id, uint32_t layerIndex, uint32_t* ptr_active_indices, uint32_t num_active_indices);
-    void propagateSync(const uint32_t requestConfigId);
+    void propagateSync(const uint32_t requestConfigId, Gna2AccelerationMode gna2AccelerationMode);
-    uint32_t propagate(const uint32_t requestConfigId);
+    uint32_t propagate(const uint32_t requestConfigId, Gna2AccelerationMode gna2AccelerationMode);
 #if GNA_LIB_VER == 2
     uint32_t createModel(const Gna2Model& gnaModel) const;
 #else
     uint32_t createModel(const intel_nnet_type_t& intel_nnet_type);
 #endif
-    void releseModel(const uint32_t model_id);
+    void releaseModel(const uint32_t model_id);
     uint32_t createRequestConfig(const uint32_t model_id);
     bool hasGnaHw() const {
         return Gna2DeviceVersionSoftwareEmulation != detectedGnaDevVersion;

inference-engine/src/gna_plugin/gna_executable_network.hpp

@@ -12,6 +12,7 @@
 #include <cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp>
 #include "gna_infer_request.hpp"
 #include "gna_plugin.hpp"
+#include <gna/gna_config.hpp>
 #include <threading/ie_executor_manager.hpp>
 #include <cpp_interfaces/impl/ie_executable_network_thread_safe_async_only.hpp>
 
@@ -67,6 +68,36 @@ class GNAExecutableNetwork : public InferenceEngine::ExecutableNetworkThreadSafe
         THROW_IE_EXCEPTION << NOT_IMPLEMENTED_str;
     }
 
+    void SetConfig(const std::map<std::string, InferenceEngine::Parameter>& config,
+                   InferenceEngine::ResponseDesc* /* resp */) override {
+        using namespace InferenceEngine::GNAConfigParams;
+        if (config.empty()) {
+            THROW_IE_EXCEPTION << "The list of configuration values is empty";
+        }
+        for (auto&& item : config) {
+            if (item.first != KEY_GNA_DEVICE_MODE) {
+                THROW_IE_EXCEPTION << "The following config value cannot be changed dynamically for ExecutableNetwork in the GNA plugin: "
+                                   << item.first << ". Only " << KEY_GNA_DEVICE_MODE << " is supported.";
+            }
+        }
+
+        InferenceEngine::Parameter old_mode_parameter;
+        GetConfig(KEY_GNA_DEVICE_MODE, old_mode_parameter, {});
+        auto old_mode = old_mode_parameter.as<std::string>();
+        if (old_mode == InferenceEngine::GNAConfigParams::GNA_SW_FP32) {
+            THROW_IE_EXCEPTION << "Dynamic switching from GNA_SW_FP32 mode is not supported for ExecutableNetwork.";
+        }
+
+        auto new_mode = config.begin()->second.as<std::string>();
+        if (new_mode == InferenceEngine::GNAConfigParams::GNA_SW_FP32) {
+            THROW_IE_EXCEPTION << "Dynamic switching to GNA_SW_FP32 mode is not supported for ExecutableNetwork.";
+        }
+
+        std::map<std::string, std::string> configForPlugin;
+        configForPlugin[KEY_GNA_DEVICE_MODE] = new_mode;
+        plg->SetConfig(configForPlugin);
+    }
+
     void GetConfig(const std::string &name,
                    InferenceEngine::Parameter &result,
                    InferenceEngine::ResponseDesc* /*resp*/) const override {

inference-engine/src/gna_plugin/gna_plugin.cpp

@@ -323,13 +323,11 @@ void GNAPlugin::Init() {
 
 void GNAPlugin::InitGNADevice() {
 #if GNA_LIB_VER == 1
-    gnadevice = std::make_shared<GNADeviceHelper>(config.gna_proc_type,
+    gnadevice = std::make_shared<GNADeviceHelper>(gnaFlags->gna_lib_async_threads_num,
-                                        gnaFlags->gna_lib_async_threads_num,
+                                                  gnaFlags->gna_openmp_multithreading,
-                                        gnaFlags->gna_openmp_multithreading,
+                                                  gnaFlags->performance_counting);
-                                        gnaFlags->performance_counting);
 #else
-    gnadevice = std::make_shared<GNADeviceHelper>(config.pluginGna2AccMode,
+    gnadevice = std::make_shared<GNADeviceHelper>(config.pluginGna2DeviceConsistent,
-                                                  config.pluginGna2DeviceConsistent,
                 gnaFlags->gna_lib_async_threads_num,
                 gnaFlags->gna_openmp_multithreading,
                 gnaFlags->performance_counting);
@@ -811,7 +809,7 @@ void GNAPlugin::DumpXNNToFile() const {
         gnadevice->dumpXnnForDeviceVersion(modelId, dumpStream,
             *reinterpret_cast<const Gna2DeviceVersion*>(&versionInt));
     }
-    gnadevice->releseModel(modelId);
+    gnadevice->releaseModel(modelId);
 #endif
 }
 
@@ -934,12 +932,12 @@ uint32_t GNAPlugin::QueueInference(const InferenceEngine::BlobMap &inputs, Infer
     } else {
 #if GNA_LIB_VER == 1
         auto nnet = std::get<0>(*freeNnet).get();
-        std::get<1>(*freeNnet) = gnadevice->propagate(&nnet->obj, ptr_active_indices, num_active_indices);
+        std::get<1>(*freeNnet) = gnadevice->propagate(&nnet->obj, ptr_active_indices, num_active_indices, config.gna_proc_type);
 #else
         const auto reqConfigId = std::get<0>(*freeNnet);
         if (ptr_active_indices != nullptr && num_active_indices > 0 && activeLayerIndex != 0xffffffff)
             gnadevice->setUpActiveList(reqConfigId, activeLayerIndex, ptr_active_indices, num_active_indices);
-        std::get<1>(*freeNnet) = gnadevice->propagate(reqConfigId);
+        std::get<1>(*freeNnet) = gnadevice->propagate(reqConfigId, config.pluginGna2AccMode);
 #endif
     }
 

inference-engine/src/gna_plugin/gna_plugin_query_api.cpp

@@ -67,24 +67,16 @@ Parameter GNAPlugin::GetAvailableDevices() const {
     std::vector<std::string> devices;
     // probing for gna-sw-exact, or gna-sw implementation part of libgna
     try {
-#if GNA_LIB_VER == 2
+        GNADeviceHelper swHelper;
-        GNADeviceHelper swHelper(Gna2AccelerationModeSoftware);
-#else
-        GNADeviceHelper swHelper(GNA_SOFTWARE);
-#endif
         devices.push_back("GNA_SW");
     }catch(...) {}
 
     try {
-#if GNA_LIB_VER == 2
+        GNADeviceHelper hwHelper;
-        GNADeviceHelper hwHelper(Gna2AccelerationModeHardware);
-#else
-        GNADeviceHelper hwHelper(GNA_HARDWARE);
-#endif
 #if GNA_LIB_VER == 1
         try {
             intel_nnet_type_t neuralNetwork = { 0 };
-            hwHelper.propagate(&neuralNetwork, nullptr, 0);
+            hwHelper.propagate(&neuralNetwork, nullptr, 0, GNA_HARDWARE);
         }catch (...) {
             if (hwHelper.getGNAStatus() != GNA_DEVNOTFOUND) {
                 devices.push_back("GNA_HW");

inference-engine/tests_deprecated/functional/gna/shared_tests_instance/ie_class/ie_class.cpp

@@ -3,12 +3,7 @@
 //
 
 #include "ie_class.hpp"
-
+#include <gna/gna_config.hpp>
-// Copyright (C) 2018-2020 Intel Corporation
-// SPDX-License-Identifier: Apache-2.0
-//
-
-#include "ie_class.hpp"
 
 //
 // IE Class Common tests with <pluginName, deviceName params>
@@ -110,6 +105,42 @@ INSTANTIATE_TEST_CASE_P(
     IEClassExecutableNetworkSetConfigTest, IEClassExecutableNetworkSetConfigTest,
     ::testing::Values("GNA"));
 
+INSTANTIATE_TEST_CASE_P(
+    IEClassExecutableNetworkSupportedConfigTest, IEClassExecutableNetworkSupportedConfigTest,
+    ::testing::Combine(::testing::Values("GNA"),
+                       ::testing::Values(std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_HW),
+                                         std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW),
+                                         std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW_EXACT),
+                                         std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_AUTO))));
+
+INSTANTIATE_TEST_CASE_P(
+    IEClassExecutableNetworkUnsupportedConfigTest, IEClassExecutableNetworkUnsupportedConfigTest,
+    ::testing::Combine(::testing::Values("GNA"),
+                       ::testing::Values(std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW_FP32),
+                                         std::make_pair(GNA_CONFIG_KEY(SCALE_FACTOR), "5"),
+                                         std::make_pair(CONFIG_KEY(EXCLUSIVE_ASYNC_REQUESTS), CONFIG_VALUE(YES)),
+                                         std::make_pair(GNA_CONFIG_KEY(COMPACT_MODE), CONFIG_VALUE(NO)))));
+
+using IEClassExecutableNetworkSetConfigFromFp32Test = IEClassExecutableNetworkGetMetricTestForSpecificConfig;
+TEST_P(IEClassExecutableNetworkSetConfigFromFp32Test, SetConfigFromFp32Throws) {
+    Core ie;
+
+    std::map<std::string, std::string> initialConfig;
+    initialConfig[GNA_CONFIG_KEY(DEVICE_MODE)] = GNAConfigParams::GNA_SW_FP32;
+    ExecutableNetwork exeNetwork = ie.LoadNetwork(simpleNetwork, deviceName, initialConfig);
+
+    ASSERT_THROW(exeNetwork.SetConfig({ { configKey, configValue } }), InferenceEngineException);
+}
+
+INSTANTIATE_TEST_CASE_P(
+    IEClassExecutableNetworkSetConfigFromFp32Test, IEClassExecutableNetworkSetConfigFromFp32Test,
+    ::testing::Combine(::testing::Values("GNA"),
+                       ::testing::Values(std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_HW),
+                                         std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW),
+                                         std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW_EXACT),
+                                         std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_SW_FP32),
+                                         std::make_pair(GNA_CONFIG_KEY(DEVICE_MODE), GNAConfigParams::GNA_AUTO))));
+
 // IE Class Query network
 
 INSTANTIATE_TEST_CASE_P(

inference-engine/tests_deprecated/functional/shared_tests/ie_class/ie_class.hpp

@@ -785,6 +785,21 @@ public:
     }
 };
 
+class IEClassExecutableNetworkGetMetricTestForSpecificConfig : public IEClassNetworkTest,
+public WithParamInterface<std::tuple<std::string, std::pair<std::string, std::string>>> {
+protected:
+    std::string deviceName;
+    std::string configKey;
+    std::string configValue;
+public:
+    virtual void SetUp() {
+        IEClassNetworkTest::SetUp();
+        deviceName = get<0>(GetParam());
+        configKey = get<1>(GetParam()).first;
+        configValue = get<1>(GetParam()).second;
+    }
+};
+
 #define ASSERT_EXEC_METRIC_SUPPORTED(metricName)                         \
     {                                                                    \
         std::vector<std::string> metrics =                               \
@@ -907,6 +922,27 @@ TEST_P(IEClassExecutableNetworkSetConfigTest, SetConfigThrows) {
     ASSERT_THROW(exeNetwork.SetConfig({ { "unsupported_config", "some_value" } }), InferenceEngineException);
 }
 
+using IEClassExecutableNetworkSupportedConfigTest = IEClassExecutableNetworkGetMetricTestForSpecificConfig;
+TEST_P(IEClassExecutableNetworkSupportedConfigTest, SupportedConfigWorks) {
+    Core ie;
+    Parameter p;
+
+    ExecutableNetwork exeNetwork = ie.LoadNetwork(simpleNetwork, deviceName);
+
+    ASSERT_NO_THROW(exeNetwork.SetConfig({ { configKey, configValue } }));
+    ASSERT_NO_THROW(p = exeNetwork.GetConfig( configKey ));
+    ASSERT_EQ(p, configValue);
+}
+
+using IEClassExecutableNetworkUnsupportedConfigTest = IEClassExecutableNetworkGetMetricTestForSpecificConfig;
+TEST_P(IEClassExecutableNetworkUnsupportedConfigTest, UnsupportedConfigThrows) {
+    Core ie;
+
+    ExecutableNetwork exeNetwork = ie.LoadNetwork(simpleNetwork, deviceName);
+
+    ASSERT_THROW(exeNetwork.SetConfig({ { configKey, configValue } }), InferenceEngineException);
+}
+
 using IEClassExecutableNetworkGetConfigTest = IEClassExecutableNetworkGetMetricTest;
 TEST_P(IEClassExecutableNetworkGetConfigTest, GetConfigNoEmptyNoThrow) {
     Core ie;