[Benchmark] benchmark_app need disable auto batching if batch size is set. (#12553)

* When set batch_size, disable auto batching

* auto-batch only supprt auto plugin

* Set property allow_auto_batching for devices

* cancel static

* Clean Code

* Add test case for testing allow_auto_batching

* Fix the issue after merging latest master branch code

* Add allow_auto_batching in python benchmark_app

* Fix the quotes issue

* When load_network is set to allow-auto-batching, there is no need to read the value of global

* Fix the clang-format

* Change the location

Co-authored-by: River Li <river.li@intel.com>
Co-authored-by: Chen Peter <peter.chen@intel.com>

samples/cpp/benchmark_app/main.cpp

@@ -437,6 +437,11 @@ int main(int argc, char* argv[]) {
             core.set_property(ov::cache_dir(FLAGS_cache_dir));
         }
 
+        // If set batch size, disable the auto batching
+        if (FLAGS_b > 0) {
+            core.set_property(ov::hint::allow_auto_batching(false));
+        }
+
         bool isDynamicNetwork = false;
 
         if (FLAGS_load_from_file && !isNetworkCompiled) {

src/bindings/python/tests/test_runtime/test_properties.py

@@ -28,6 +28,15 @@ def test_property_ro():
     assert "available_devices(): incompatible function arguments." in str(e.value)
 
 
+def test_allow_auto_batching_property():
+    core = Core()
+    core.set_property({"ALLOW_AUTO_BATCHING": False})
+    assert core.get_property(properties.hint.allow_auto_batching()) is False
+
+    core.set_property({"ALLOW_AUTO_BATCHING": True})
+    assert core.get_property(properties.hint.allow_auto_batching()) is True
+
+
 @pytest.mark.skipif(os.environ.get("TEST_DEVICE", "CPU") != "CPU",
                     reason=f"Cannot run test on device {os.environ.get('TEST_DEVICE')}, Plugin specific test")
 def test_single_property_setting():

src/inference/src/ie_core.cpp

@@ -243,6 +243,8 @@ class CoreImpl : public ie::ICore, public std::enable_shared_from_this<ie::ICore
             std::shared_ptr<ie::ICacheManager> _cacheManager;
         };
 
+        bool flag_allow_auto_batching = true;
+
         void setAndUpdate(ov::AnyMap& config) {
             auto it = config.find(CONFIG_KEY(CACHE_DIR));
             if (it != config.end()) {
@@ -260,6 +262,13 @@ class CoreImpl : public ie::ICore, public std::enable_shared_from_this<ie::ICore
                 executorManager()->setTbbFlag(flag);
                 config.erase(it);
             }
+
+            it = config.find(ov::hint::allow_auto_batching.name());
+            if (it != config.end()) {
+                auto flag = it->second.as<bool>();
+                flag_allow_auto_batching = flag;
+                config.erase(it);
+            }
         }
 
         void setCacheForDevice(const std::string& dir, const std::string& name) {
@@ -746,6 +755,8 @@ public:
                     config.erase(batch_mode);
                 if (disabled)
                     return;
+            } else if (!coreConfig.flag_allow_auto_batching) {
+                return;
             }
             // check whether if the Auto-Batching is applicable to the device
             auto device = ov::parseDeviceNameIntoConfig(deviceName);
@@ -987,6 +998,9 @@ public:
             return decltype(ov::force_tbb_terminate)::value_type(flag);
         } else if (name == ov::cache_dir.name()) {
             return ov::Any(coreConfig.get_cache_dir());
+        } else if (name == ov::hint::allow_auto_batching.name()) {
+            const auto flag = coreConfig.flag_allow_auto_batching;
+            return decltype(ov::hint::allow_auto_batching)::value_type(flag);
         }
 
         IE_THROW() << "Exception is thrown while trying to call get_property with unsupported property: '" << name

src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp

@@ -433,6 +433,20 @@ TEST(OVClassBasicTest, GetUnsupportedPropertyCoreThrow) {
     ASSERT_THROW(ie.get_property("unsupported_property"), ov::Exception);
 }
 
+TEST(OVClassBasicTest, SetAllowAutoBatchingPropertyCoreNoThrows) {
+    ov::Core ie = createCoreWithTemplate();
+
+    bool value1 = true;
+    OV_ASSERT_NO_THROW(ie.set_property(ov::hint::allow_auto_batching(false)));
+    OV_ASSERT_NO_THROW(value1 = ie.get_property(ov::hint::allow_auto_batching.name()));
+    ASSERT_FALSE(value1);
+
+    bool value2 = false;
+    OV_ASSERT_NO_THROW(ie.set_property(ov::hint::allow_auto_batching(true)));
+    OV_ASSERT_NO_THROW(value2 = ie.get_property(ov::hint::allow_auto_batching.name()));
+    ASSERT_TRUE(value2);
+}
+
 TEST_P(OVClassSetLogLevelConfigTest, SetConfigNoThrow) {
     ov::Core ie = createCoreWithTemplate();
     // log level

tools/benchmark_tool/openvino/tools/benchmark/benchmark.py

@@ -55,6 +55,9 @@ class Benchmark:
     def set_cache_dir(self, cache_dir: str):
         self.core.set_property({'CACHE_DIR': cache_dir})
 
+    def set_allow_auto_batching(self, flag: bool):
+        self.core.set_property({'ALLOW_AUTO_BATCHING': flag})
+
     def read_model(self, path_to_model: str):
         model_filename = os.path.abspath(path_to_model)
         head, ext = os.path.splitext(model_filename)

tools/benchmark_tool/openvino/tools/benchmark/main.py

@@ -242,6 +242,10 @@ def main():
         if args.cache_dir:
             benchmark.set_cache_dir(args.cache_dir)
 
+        ## If set batch size, disable the auto batching
+        if args.batch_size:
+            benchmark.set_allow_auto_batching(False)
+
         topology_name = ""
         load_from_file_enabled = is_flag_set_in_command_line('load_from_file') or is_flag_set_in_command_line('lfile')
         if load_from_file_enabled and not is_network_compiled: