IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Do not clean batch setting if proxy plugin (#19508)

Browse Source 
* do not clean batch setting if proxy plugin

Signed-off-by: fishbell <bell.song@intel.com>

* add tests

Signed-off-by: fishbell <bell.song@intel.com>

---------

Signed-off-by: fishbell <bell.song@intel.com>
This commit is contained in:
yanlan song 2023-09-07 11:47:45 +08:00 committed by GitHub
parent 63a6d4c41e
commit 14e0b1fd2c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 38 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/inference/src/dev/core_impl.cpp
View File

@ -743,7 +743,7 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::compile_model(const std::shared_ptr<
// if auto-batching is applicable, the below function will patch the device name and config accordingly: // if auto-batching is applicable, the below function will patch the device name and config accordingly:
auto model = apply_auto_batching(model_, deviceName, config_with_batch); auto model = apply_auto_batching(model_, deviceName, config_with_batch);
auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch); auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch, is_proxy_device(device_name));
auto plugin = get_plugin(parsed._deviceName); auto plugin = get_plugin(parsed._deviceName);
ov::SoPtr<ov::ICompiledModel> res; ov::SoPtr<ov::ICompiledModel> res;
auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager;
@ -776,7 +776,7 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::compile_model(const std::shared_ptr<
// if auto-batching is applicable, the below function will patch the device name and config accordingly: // if auto-batching is applicable, the below function will patch the device name and config accordingly:
auto model = apply_auto_batching(model_, deviceName, config_with_batch); auto model = apply_auto_batching(model_, deviceName, config_with_batch);
auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch); auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch, is_proxy_device(deviceName));
auto plugin = get_plugin(parsed._deviceName); auto plugin = get_plugin(parsed._deviceName);
ov::SoPtr<ov::ICompiledModel> res; ov::SoPtr<ov::ICompiledModel> res;
auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager;
@ -1095,8 +1095,9 @@ std::shared_ptr<const ov::Model> ov::CoreImpl::apply_auto_batching(const std::sh
const auto disabled = batch_mode->second.as<std::string>() == CONFIG_VALUE(NO); const auto disabled = batch_mode->second.as<std::string>() == CONFIG_VALUE(NO);
// virtual plugins like AUTO/MULTI will need the config // virtual plugins like AUTO/MULTI will need the config
// e.g. to deduce the #requests correctly // e.g. to deduce the #requests correctly
// proxy plugin should also keep the config
// otherwise, no need for this config key in the rest of loading // otherwise, no need for this config key in the rest of loading
if (!is_virtual_device(deviceName)) if (!is_virtual_device(deviceName) && !is_proxy_device(deviceName))
config.erase(batch_mode); config.erase(batch_mode);
if (disabled) if (disabled)
return model; return model;

34
src/plugins/proxy/tests/batch_compliance_test.cpp Normal file
View File

@ -0,0 +1,34 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "openvino/proxy/properties.hpp"
#include "openvino/runtime/properties.hpp"
#include "proxy_tests.hpp"
using namespace ov::proxy::tests;
TEST_F(ProxyTests, can_parse_and_inherit_batch_property) {
register_plugin_support_reshape(core, "MOCK_DEVICE", {{ov::proxy::configuration::alias.name(), "ALIAS_MOCK"}});
auto available_devices = core.get_available_devices();
auto model = create_model_with_add();
auto compiled_model_default = core.compile_model(model, "MOCK_DEVICE", ov::hint::performance_mode("THROUGHPUT"));
#ifdef ENABLE_AUTO_BATCH
EXPECT_NO_THROW(compiled_model_default.get_property(ov::auto_batch_timeout)); // batch enabled by default
EXPECT_EQ(compiled_model_default.get_property(ov::auto_batch_timeout), 1000); // default value
#endif
auto compiled_model_with_batch = core.compile_model(model,
"MOCK_DEVICE",
ov::hint::performance_mode("THROUGHPUT"),
ov::hint::allow_auto_batching(true),
ov::auto_batch_timeout(8));
#ifdef ENABLE_AUTO_BATCH
EXPECT_NO_THROW(compiled_model_with_batch.get_property(ov::auto_batch_timeout));
EXPECT_EQ(compiled_model_with_batch.get_property(ov::auto_batch_timeout), 8);
#endif
auto compiled_model_no_batch = core.compile_model(model,
"MOCK_DEVICE",
ov::hint::performance_mode("THROUGHPUT"),
ov::hint::allow_auto_batching(false));
EXPECT_ANY_THROW(compiled_model_no_batch.get_property(ov::auto_batch_timeout));
}