AUTO/MULTI supports ov::auto_batch_timeout (#12023)

* add auto_batch_timeout for MULTI and AUTO

* fix clang-format for ie_core.cpp

* fix coredump

* simplify insert key to deviceConfig logic and parseDeviceNameIntoConfig() check "AUTO" and "AUTO:" only

* check config auto_batch_timeout

* add CleanUpInIECore()

* fix clang-format for ie_core.cpp

src/inference/src/ie_core.cpp

@@ -96,7 +96,7 @@ Parsed<T> parseDeviceNameIntoConfig(const std::string& deviceName, const std::ma
     } else if (deviceName_.find("MULTI:") == 0) {
         deviceName_ = "MULTI";
         config_[ie::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES] = deviceName.substr(6);
-    } else if (deviceName.find("AUTO") == 0) {
+    } else if (deviceName == "AUTO" || deviceName.find("AUTO:") == 0) {
         deviceName_ = "AUTO";
         if (deviceName.find("AUTO:") == 0) {
             config_[ie::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES] =
@@ -615,6 +615,7 @@ public:
         std::map<std::string, std::string>& config_with_batch = parsed._config;
         // if auto-batching is applicable, the below function will patch the device name and config accordingly:
         ApplyAutoBatching(network, deviceName, config_with_batch);
+        CleanUpProperties(deviceName, config_with_batch);
         parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch);
 
         auto plugin = GetCPPPluginByName(parsed._deviceName);
@@ -701,6 +702,17 @@ public:
         }
     }
 
+    void CleanUpProperties(std::string& deviceName, std::map<std::string, std::string>& config) {
+        // auto-batching is not applicable, if there is auto_batch_timeout, delete it
+        if (deviceName.find("BATCH") == std::string::npos) {
+            const auto& batch_timeout_mode = config.find(ov::auto_batch_timeout.name());
+            if (batch_timeout_mode != config.end()) {
+                if (deviceName.find("AUTO") == std::string::npos && deviceName.find("MULTI") == std::string::npos)
+                    config.erase(batch_timeout_mode);
+            }
+        }
+    }
+
     ie::SoExecutableNetworkInternal LoadNetwork(const ie::CNNNetwork& network,
                                                 const std::string& deviceNameOrig,
                                                 const std::map<std::string, std::string>& config) override {
@@ -709,6 +721,7 @@ public:
         std::map<std::string, std::string> config_with_batch = config;
         // if auto-batching is applicable, the below function will patch the device name and config accordingly:
         ApplyAutoBatching(network, deviceName, config_with_batch);
+        CleanUpProperties(deviceName, config_with_batch);
 
         bool forceDisableCache = config_with_batch.count(CONFIG_KEY_INTERNAL(FORCE_DISABLE_CACHE)) > 0;
         auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch);

src/plugins/auto/plugin.cpp

@@ -69,6 +69,7 @@ namespace {
                     res.push_back(ov::hint::allow_auto_batching.name());
                     res.push_back(ov::log::level.name());
                     res.push_back(ov::intel_auto::device_bind_buffer.name());
+                    res.push_back(ov::auto_batch_timeout.name());
                     return res;
                 }();
 }  // namespace
@@ -269,6 +270,7 @@ InferenceEngine::Parameter MultiDeviceInferencePlugin::GetMetric(const std::stri
                                                     RW_property(ov::device::priorities.name()),
                                                     RW_property(ov::enable_profiling.name()),
                                                     RW_property(ov::hint::allow_auto_batching.name()),
+                                                    RW_property(ov::auto_batch_timeout.name()),
                                                     RW_property(ov::hint::performance_mode.name()),
                                                     RW_property(ov::hint::num_requests.name())
         };
@@ -343,7 +345,25 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
     std::vector<DeviceInformation> metaDevices;
     bool workModeAuto = GetName() == "AUTO";
     auto priorities = fullConfig.find(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES);
+    // If the user sets the property, insert the property into the deviceConfig
+    auto insertPropToConfig = [&](std::string property,
+                                  std::string& deviceName,
+                                  std::map<std::string, std::string>& deviceConfig) {
+        auto tmpiter =
+            std::find_if(fullConfig.begin(), fullConfig.end(), [&](const std::pair<std::string, std::string>& config) {
+                return (config.first == property);
+            });
+        if (tmpiter != fullConfig.end()) {
+            deviceConfig.insert({tmpiter->first, tmpiter->second});
+            LOG_INFO_TAG("device:%s, config:%s=%s",
+                         deviceName.c_str(),
+                         tmpiter->first.c_str(),
+                         tmpiter->second.c_str());
+        }
+    };
+
     // if workMode is AUTO
+    // only AUTO uses CheckConfig() to check fullConfig's parameters, MULTI does not
     if (workModeAuto) {
         // check the configure and check if need to set PerfCounters configure to device
         // and set filter configure
@@ -396,11 +416,8 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
                              config.first.c_str(), config.second.c_str());
                  }
              }
-             auto tmpiter = std::find_if(fullConfig.begin(), fullConfig.end(), [](const std::pair<std::string, std::string>& config) {
+             insertPropToConfig(CONFIG_KEY(ALLOW_AUTO_BATCHING), iter->deviceName, deviceConfig);
-                            return (config.first == CONFIG_KEY(ALLOW_AUTO_BATCHING));
+             insertPropToConfig(CONFIG_KEY(AUTO_BATCH_TIMEOUT), iter->deviceName, deviceConfig);
-                            });
-             if (tmpiter != fullConfig.end())
-                 deviceConfig.insert({tmpiter->first, tmpiter->second});
              iter->config = deviceConfig;
              strDevices += iter->deviceName;
              strDevices += ((iter + 1) == supportDevices.end()) ? "" : ",";
@@ -450,6 +467,7 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
                     multiSContext->_batchingDisabled = true;
                 p.config.insert({tmpiter->first, tmpiter->second});
             }
+            insertPropToConfig(CONFIG_KEY(AUTO_BATCH_TIMEOUT), p.deviceName, p.config);
             const auto& deviceName = p.deviceName;
             const auto& deviceConfig = p.config;
             SoExecutableNetworkInternal exec_net;
@@ -846,6 +864,17 @@ void MultiDeviceInferencePlugin::CheckConfig(const std::map<std::string, std::st
                 context->_batchingDisabled = true;
                 continue;
             }
+        } else if (kvp.first == ov::auto_batch_timeout) {
+            try {
+                auto batch_timeout = std::stoi(kvp.second);
+                if (batch_timeout < 0) {
+                    IE_THROW() << "Unsupported config value: " << kvp.second
+                           << " for key: " << kvp.first;
+                }
+            } catch (...) {
+                IE_THROW() << "Unsupported config value: " << kvp.second
+                           << " for key: " << kvp.first;
+            }
         } else if (kvp.first == ov::intel_auto::device_bind_buffer.name()) {
             if (kvp.second == PluginConfigParams::YES ||
                 kvp.second == PluginConfigParams::NO) {