From 9d56cfd79b58e0375c9c1be61abcc0422d36a134 Mon Sep 17 00:00:00 2001 From: RICKIE777 Date: Thu, 22 Sep 2022 09:50:30 +0800 Subject: [PATCH] [Benchmark] benchmark_app need disable auto batching if batch size is set. (#12553) * When set batch_size, disable auto batching * auto-batch only supprt auto plugin * Set property allow_auto_batching for devices * cancel static * Clean Code * Add test case for testing allow_auto_batching * Fix the issue after merging latest master branch code * Add allow_auto_batching in python benchmark_app * Fix the quotes issue * When load_network is set to allow-auto-batching, there is no need to read the value of global * Fix the clang-format * Change the location Co-authored-by: River Li Co-authored-by: Chen Peter --- samples/cpp/benchmark_app/main.cpp | 5 +++++ .../python/tests/test_runtime/test_properties.py | 9 +++++++++ src/inference/src/ie_core.cpp | 14 ++++++++++++++ .../behavior/ov_plugin/core_integration.hpp | 14 ++++++++++++++ .../openvino/tools/benchmark/benchmark.py | 3 +++ .../openvino/tools/benchmark/main.py | 4 ++++ 6 files changed, 49 insertions(+) diff --git a/samples/cpp/benchmark_app/main.cpp b/samples/cpp/benchmark_app/main.cpp index 728f2ce8702..5150d7e9499 100644 --- a/samples/cpp/benchmark_app/main.cpp +++ b/samples/cpp/benchmark_app/main.cpp @@ -437,6 +437,11 @@ int main(int argc, char* argv[]) { core.set_property(ov::cache_dir(FLAGS_cache_dir)); } + // If set batch size, disable the auto batching + if (FLAGS_b > 0) { + core.set_property(ov::hint::allow_auto_batching(false)); + } + bool isDynamicNetwork = false; if (FLAGS_load_from_file && !isNetworkCompiled) { diff --git a/src/bindings/python/tests/test_runtime/test_properties.py b/src/bindings/python/tests/test_runtime/test_properties.py index b2ddc1a7e9e..0200fab4078 100644 --- a/src/bindings/python/tests/test_runtime/test_properties.py +++ b/src/bindings/python/tests/test_runtime/test_properties.py @@ -28,6 +28,15 @@ def test_property_ro(): assert "available_devices(): incompatible function arguments." in str(e.value) +def test_allow_auto_batching_property(): + core = Core() + core.set_property({"ALLOW_AUTO_BATCHING": False}) + assert core.get_property(properties.hint.allow_auto_batching()) is False + + core.set_property({"ALLOW_AUTO_BATCHING": True}) + assert core.get_property(properties.hint.allow_auto_batching()) is True + + @pytest.mark.skipif(os.environ.get("TEST_DEVICE", "CPU") != "CPU", reason=f"Cannot run test on device {os.environ.get('TEST_DEVICE')}, Plugin specific test") def test_single_property_setting(): diff --git a/src/inference/src/ie_core.cpp b/src/inference/src/ie_core.cpp index d7ce4deb8a9..cb0901f1a1a 100644 --- a/src/inference/src/ie_core.cpp +++ b/src/inference/src/ie_core.cpp @@ -243,6 +243,8 @@ class CoreImpl : public ie::ICore, public std::enable_shared_from_this _cacheManager; }; + bool flag_allow_auto_batching = true; + void setAndUpdate(ov::AnyMap& config) { auto it = config.find(CONFIG_KEY(CACHE_DIR)); if (it != config.end()) { @@ -260,6 +262,13 @@ class CoreImpl : public ie::ICore, public std::enable_shared_from_thissetTbbFlag(flag); config.erase(it); } + + it = config.find(ov::hint::allow_auto_batching.name()); + if (it != config.end()) { + auto flag = it->second.as(); + flag_allow_auto_batching = flag; + config.erase(it); + } } void setCacheForDevice(const std::string& dir, const std::string& name) { @@ -746,6 +755,8 @@ public: config.erase(batch_mode); if (disabled) return; + } else if (!coreConfig.flag_allow_auto_batching) { + return; } // check whether if the Auto-Batching is applicable to the device auto device = ov::parseDeviceNameIntoConfig(deviceName); @@ -987,6 +998,9 @@ public: return decltype(ov::force_tbb_terminate)::value_type(flag); } else if (name == ov::cache_dir.name()) { return ov::Any(coreConfig.get_cache_dir()); + } else if (name == ov::hint::allow_auto_batching.name()) { + const auto flag = coreConfig.flag_allow_auto_batching; + return decltype(ov::hint::allow_auto_batching)::value_type(flag); } IE_THROW() << "Exception is thrown while trying to call get_property with unsupported property: '" << name diff --git a/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp index 99a10a32d4d..d0a8b138a76 100644 --- a/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp +++ b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp @@ -433,6 +433,20 @@ TEST(OVClassBasicTest, GetUnsupportedPropertyCoreThrow) { ASSERT_THROW(ie.get_property("unsupported_property"), ov::Exception); } +TEST(OVClassBasicTest, SetAllowAutoBatchingPropertyCoreNoThrows) { + ov::Core ie = createCoreWithTemplate(); + + bool value1 = true; + OV_ASSERT_NO_THROW(ie.set_property(ov::hint::allow_auto_batching(false))); + OV_ASSERT_NO_THROW(value1 = ie.get_property(ov::hint::allow_auto_batching.name())); + ASSERT_FALSE(value1); + + bool value2 = false; + OV_ASSERT_NO_THROW(ie.set_property(ov::hint::allow_auto_batching(true))); + OV_ASSERT_NO_THROW(value2 = ie.get_property(ov::hint::allow_auto_batching.name())); + ASSERT_TRUE(value2); +} + TEST_P(OVClassSetLogLevelConfigTest, SetConfigNoThrow) { ov::Core ie = createCoreWithTemplate(); // log level diff --git a/tools/benchmark_tool/openvino/tools/benchmark/benchmark.py b/tools/benchmark_tool/openvino/tools/benchmark/benchmark.py index 99f8d6b9ba9..26829bf920e 100644 --- a/tools/benchmark_tool/openvino/tools/benchmark/benchmark.py +++ b/tools/benchmark_tool/openvino/tools/benchmark/benchmark.py @@ -55,6 +55,9 @@ class Benchmark: def set_cache_dir(self, cache_dir: str): self.core.set_property({'CACHE_DIR': cache_dir}) + def set_allow_auto_batching(self, flag: bool): + self.core.set_property({'ALLOW_AUTO_BATCHING': flag}) + def read_model(self, path_to_model: str): model_filename = os.path.abspath(path_to_model) head, ext = os.path.splitext(model_filename) diff --git a/tools/benchmark_tool/openvino/tools/benchmark/main.py b/tools/benchmark_tool/openvino/tools/benchmark/main.py index 73b6a2d76be..0e38988e478 100644 --- a/tools/benchmark_tool/openvino/tools/benchmark/main.py +++ b/tools/benchmark_tool/openvino/tools/benchmark/main.py @@ -242,6 +242,10 @@ def main(): if args.cache_dir: benchmark.set_cache_dir(args.cache_dir) + ## If set batch size, disable the auto batching + if args.batch_size: + benchmark.set_allow_auto_batching(False) + topology_name = "" load_from_file_enabled = is_flag_set_in_command_line('load_from_file') or is_flag_set_in_command_line('lfile') if load_from_file_enabled and not is_network_compiled: