IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[apiConformance] Add tests on props (#15272)

Browse Source
This commit is contained in:
Sofya Balandina 2023-01-31 14:52:01 +00:00 committed by GitHub
parent a12de8183c
commit 12a9621650
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 248 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/properties.cpp
View File

@ -54,4 +54,26 @@ INSTANTIATE_TEST_SUITE_P(ov_plugin_AutoBatch, OVPropertiesTests,
::testing::Values(CommonTestUtils::DEVICE_BATCH),
::testing::ValuesIn(ov::test::conformance::generate_ov_configs(CommonTestUtils::DEVICE_BATCH, auto_batch_properties))),
OVPropertiesTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(ov_plugin, OVCheckChangePropComplieModleGetPropTests,
::testing::Combine(
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(OVCheckChangePropComplieModleGetPropTests::getPropertiesValues())),
OVCheckChangePropComplieModleGetPropTests::getTestCaseName);
const std::vector<ov::AnyMap> device_properties = {
{ov::device::id("0")},
};
INSTANTIATE_TEST_SUITE_P(ov_plugin, OVCheckChangePropComplieModleGetPropTests_DEVICE_ID,
::testing::Combine(
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(device_properties)),
OVCheckChangePropComplieModleGetPropTests_DEVICE_ID::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(ov_plugin, OVCheckChangePropComplieModleGetPropTests_ModelDependceProps,
::testing::Combine(
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(OVCheckChangePropComplieModleGetPropTests::getModelDependcePropertiesValues())),
OVCheckChangePropComplieModleGetPropTests_ModelDependceProps::getTestCaseName);
} // namespace

20
src/tests/functional/plugin/shared/include/behavior/ov_plugin/properties_tests.hpp
View File

@ -58,8 +58,28 @@ public:
AnyMap compileModelProperties;
};
using OVSetPropComplieModleWihtIncorrectPropTests = OVSetPropComplieModleGetPropTests;
class OVPropertiesTestsWithComplieModelProps : public testing::WithParamInterface<PropertiesParams>,
public OVPropertiesBase {
public:
static std::string getTestCaseName(testing::TestParamInfo<PropertiesParams> obj);
void SetUp() override;
void TearDown() override;
AnyMap compileModelProperties;
static std::vector<ov::AnyMap> getPropertiesValues();
static std::vector<ov::AnyMap> getModelDependcePropertiesValues();
};
using OVCheckChangePropComplieModleGetPropTests = OVPropertiesTestsWithComplieModelProps;
using OVCheckChangePropComplieModleGetPropTests_DEVICE_ID = OVPropertiesTestsWithComplieModelProps;
using OVCheckChangePropComplieModleGetPropTests_ModelDependceProps = OVPropertiesTestsWithComplieModelProps;
using OvPropertiesParams = std::tuple<
std::string, // device name
std::pair<ov::AnyMap, std::string> // device and expect execution device configuration

206
src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp
View File

@ -73,6 +73,52 @@ void OVSetPropComplieModleGetPropTests::SetUp() {
model = ngraph::builder::subgraph::makeConvPoolRelu();
}
std::string OVPropertiesTestsWithComplieModelProps::getTestCaseName(testing::TestParamInfo<PropertiesParams> obj) {
std::string target_device;
AnyMap properties;
std::tie(target_device, properties) = obj.param;
std::replace(target_device.begin(), target_device.end(), ':', '.');
std::ostringstream result;
result << "target_device=" << target_device << "_";
if (!properties.empty()) {
result << "properties=" << util::join(util::split(util::to_string(properties), ' '), "_");
}
return result.str();
}
void OVPropertiesTestsWithComplieModelProps::SetUp() {
SKIP_IF_CURRENT_TEST_IS_DISABLED();
std::string temp_device;
std::tie(temp_device, properties) = this->GetParam();
APIBaseTest::SetUp();
std::string::size_type pos = temp_device.find(":", 0);
std::string hw_device;
if (pos == std::string::npos) {
target_device = temp_device;
hw_device = temp_device;
} else {
target_device = temp_device.substr(0, pos);
hw_device = temp_device.substr(++pos, std::string::npos);
}
if (target_device == std::string(CommonTestUtils::DEVICE_MULTI) ||
target_device == std::string(CommonTestUtils::DEVICE_AUTO) ||
target_device == std::string(CommonTestUtils::DEVICE_HETERO)) {
compileModelProperties = { ov::device::priorities(hw_device) };
} else if (target_device == std::string(CommonTestUtils::DEVICE_BATCH)) {
compileModelProperties = {{ CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , hw_device}};
}
model = ngraph::builder::subgraph::makeConvPoolRelu();
}
void OVPropertiesTestsWithComplieModelProps::TearDown() {
if (!properties.empty()) {
utils::PluginCache::get().reset();
}
APIBaseTest::TearDown();
}
TEST_P(OVEmptyPropertiesTests, SetEmptyProperties) {
OV_ASSERT_NO_THROW(core->get_property(target_device, ov::supported_properties));
OV_ASSERT_NO_THROW(core->set_property(target_device, AnyMap{}));
@ -234,6 +280,166 @@ TEST_P(OVClassExecutableNetworkGetMetricTest_EXEC_DEVICES, CanGetExecutionDevice
ASSERT_EQ(expectedTargets, exeTargets);
}
std::vector<ov::AnyMap> OVPropertiesTestsWithComplieModelProps::getPropertiesValues() {
std::vector<ov::AnyMap> res;
// Read Only
res.push_back({{ov::PropertyName(ov::available_devices.name(), ov::available_devices.mutability), nullptr}});
res.push_back({{ov::PropertyName(ov::model_name.name(), ov::model_name.mutability), nullptr}});
res.push_back({{ov::PropertyName(ov::optimal_number_of_infer_requests.name(), ov::optimal_number_of_infer_requests.mutability), nullptr}});
res.push_back({{ov::PropertyName(ov::loaded_from_cache.name(), ov::loaded_from_cache.mutability), nullptr}});
res.push_back({{ov::PropertyName(ov::range_for_streams.name(), ov::range_for_streams.mutability), nullptr}});
res.push_back({{ov::PropertyName(ov::range_for_async_infer_requests.name(), ov::range_for_async_infer_requests.mutability), nullptr}});
res.push_back({{ov::PropertyName(ov::device::full_name.name(), ov::device::full_name.mutability), nullptr}});
res.push_back({{ov::PropertyName(ov::device::architecture.name(), ov::device::architecture.mutability), nullptr}});
res.push_back({{ov::PropertyName(ov::device::uuid.name(), ov::device::uuid.mutability), nullptr}});
res.push_back({{ov::PropertyName(ov::device::type.name(), ov::device::type.mutability), nullptr}});
res.push_back({{ov::PropertyName(ov::device::gops.name(), ov::device::gops.mutability), nullptr}});
res.push_back({{ov::PropertyName(ov::device::thermal.name(), ov::device::thermal.mutability), nullptr}});
res.push_back({{ov::PropertyName(ov::device::capabilities.name(), ov::device::capabilities.mutability), nullptr}});
res.push_back({{ov::PropertyName(ov::execution_devices.name(), ov::execution_devices.mutability), nullptr}});
res.push_back({{ov::PropertyName(ov::device::priorities.name(), ov::device::priorities.mutability), nullptr}});
// Writable
const std::vector<ov::element::Type> ovElemTypes = {
ov::element::f64, ov::element::f32, ov::element::f16,
ov::element::i64, ov::element::i32, ov::element::i16, ov::element::i8,
ov::element::u64, ov::element::u32, ov::element::u16, ov::element::u8,
ov::element::boolean,
};
for (auto &precision : ovElemTypes) {
res.push_back({{ov::PropertyName(ov::hint::inference_precision.name(), ov::hint::inference_precision.mutability), precision}});
}
ov::hint::Priority priorities[] = {ov::hint::Priority::LOW , ov::hint::Priority::MEDIUM, ov::hint::Priority::HIGH};
for (auto &priority : priorities) {
res.push_back({{ov::PropertyName(ov::hint::model_priority.name(), ov::hint::model_priority.mutability), priority}});
}
ov::hint::PerformanceMode performance_modes[] = {ov::hint::PerformanceMode::UNDEFINED , ov::hint::PerformanceMode::LATENCY,
ov::hint::PerformanceMode::THROUGHPUT, ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT};
for (auto &performance_mode : performance_modes) {
res.push_back({{ov::PropertyName(ov::hint::performance_mode.name(), ov::hint::performance_mode.mutability), performance_mode}});
}
res.push_back({{ov::PropertyName(ov::hint::num_requests.name(), ov::hint::num_requests.mutability), 1}});
res.push_back({{ov::PropertyName(ov::hint::allow_auto_batching.name(), ov::hint::allow_auto_batching.mutability), true}});
res.push_back({{ov::PropertyName(ov::hint::allow_auto_batching.name(), ov::hint::allow_auto_batching.mutability), false}});
ov::hint::ExecutionMode execution_modes[] = {ov::hint::ExecutionMode::UNDEFINED , ov::hint::ExecutionMode::PERFORMANCE,
ov::hint::ExecutionMode::ACCURACY};
for (auto &execution_mode : execution_modes) {
res.push_back({{ov::PropertyName(ov::hint::execution_mode.name(), ov::hint::execution_mode.mutability), execution_mode}});
}
res.push_back({{ov::PropertyName(ov::enable_profiling.name(), ov::enable_profiling.mutability), true}});
res.push_back({{ov::PropertyName(ov::enable_profiling.name(), ov::enable_profiling.mutability), false}});
ov::log::Level log_levels[] = {ov::log::Level::NO , ov::log::Level::ERR, ov::log::Level::WARNING,
ov::log::Level::INFO, ov::log::Level::DEBUG, ov::log::Level::TRACE};
for (auto &log_level : log_levels) {
res.push_back({{ov::PropertyName(ov::log::level.name(), ov::log::level.mutability), log_level}});
}
res.push_back({{ov::PropertyName(ov::cache_dir.name(), ov::cache_dir.mutability), ov::cache_dir("temp_cash/")}});
res.push_back({{ov::PropertyName(ov::auto_batch_timeout.name(), ov::auto_batch_timeout.mutability), 2147483647}});
res.push_back({{ov::PropertyName(ov::force_tbb_terminate.name(), ov::force_tbb_terminate.mutability), true}});
res.push_back({{ov::PropertyName(ov::force_tbb_terminate.name(), ov::force_tbb_terminate.mutability), false}});
ov::streams::Num nums[] = {ov::streams::AUTO, ov::streams::NUMA};
for (auto &num : nums) {
res.push_back({{ov::PropertyName(ov::streams::num.name(), ov::streams::num.mutability), num}});
// res.push_back({{ov::PropertyName(ov::num_streams.name(), ov::num_streams.mutability), num}});
}
res.push_back({{ov::PropertyName(ov::inference_num_threads.name(), ov::inference_num_threads.mutability), 1}});
res.push_back({{ov::PropertyName(ov::compilation_num_threads.name(), ov::compilation_num_threads.mutability), 1}});
ov::Affinity affinities[] = {ov::Affinity::NONE , ov::Affinity::CORE, ov::Affinity::NUMA, ov::Affinity::HYBRID_AWARE};
for (auto &affinity : affinities) {
res.push_back({{ov::PropertyName(ov::affinity.name(), ov::affinity.mutability), affinity}});
}
return res;
}
TEST_P(OVCheckChangePropComplieModleGetPropTests, ChangeCorrectProperties) {
std::vector<ov::PropertyName> supported_properties;
OV_ASSERT_NO_THROW(supported_properties = core->get_property(target_device, ov::supported_properties));
for (const std::pair<ov::PropertyName, ov::Any>& property_item : properties) {
auto supported = util::contains(supported_properties, property_item.first);
ASSERT_TRUE(supported) << "property is not supported: " << property_item.first;
ov::Any default_property;
OV_ASSERT_NO_THROW(default_property = core->get_property(target_device, property_item.first));
if (property_item.first.is_mutable() && !property_item.second.empty()) {
OV_ASSERT_NO_THROW(core->set_property(target_device, {property_item}));
core->compile_model(model, target_device, compileModelProperties);
ov::Any new_property_value;
OV_ASSERT_NO_THROW(new_property_value = core->get_property(target_device, property_item.first));
if (default_property != property_item.second) {
ASSERT_TRUE(new_property_value == property_item.second) << "Peoperty is not changed";
} else {
ASSERT_TRUE(new_property_value == property_item.second) << "Peoperty is changed in wrong way";
}
}
}
}
TEST_P(OVCheckChangePropComplieModleGetPropTests_DEVICE_ID, ChangeCorrectDeviceProperties) {
std::vector<ov::PropertyName> supported_properties;
OV_ASSERT_NO_THROW(supported_properties = core->get_property(target_device, ov::supported_properties));
std::vector<std::string> availableDevices = core->get_available_devices();
for (int id = 0; id < availableDevices.size(); id++) {
std::string deviceName = availableDevices[id];
auto supported = util::contains(supported_properties, ov::device::id);
ASSERT_TRUE(supported) << "property is not supported: " << ov::device::id;
ov::Any default_property;
OV_ASSERT_NO_THROW(default_property = core->get_property(target_device, ov::device::id));
OV_ASSERT_NO_THROW(core->set_property(target_device, {ov::device::id(id)}));
core->compile_model(model, target_device, compileModelProperties);
ov::Any new_property_value;
OV_ASSERT_NO_THROW(new_property_value = core->get_property(target_device, ov::device::id));
if (default_property != deviceName) {
ASSERT_TRUE(new_property_value == deviceName) << "Peoperty is not changed";
} else {
ASSERT_TRUE(new_property_value == deviceName) << "Peoperty is changed in wrong way";
}
}
}
std::vector<ov::AnyMap> OVPropertiesTestsWithComplieModelProps::getModelDependcePropertiesValues() {
std::vector<ov::AnyMap> res;
// Read Only
res.push_back({{ov::PropertyName(ov::optimal_batch_size.name(), ov::optimal_batch_size.mutability), nullptr}});
res.push_back({{ov::PropertyName(ov::max_batch_size.name(), ov::max_batch_size.mutability), nullptr}});
return res;
}
TEST_P(OVCheckChangePropComplieModleGetPropTests_ModelDependceProps, ChangeCorrectDeviceProperties) {
std::vector<ov::PropertyName> supported_properties;
OV_ASSERT_NO_THROW(supported_properties = core->get_property(target_device, ov::supported_properties));
auto supported = util::contains(supported_properties, ov::hint::model);
ASSERT_TRUE(supported) << "property is not supported: " << ov::hint::model;
OV_ASSERT_NO_THROW(core->set_property(target_device, ov::hint::model(model)));
core->compile_model(model, target_device, compileModelProperties);
for (const std::pair<ov::PropertyName, ov::Any>& property_item : properties) {
auto supported = util::contains(supported_properties, property_item.first);
ASSERT_TRUE(supported) << "property is not supported: " << property_item.first;
ov::Any default_property;
OV_ASSERT_NO_THROW(default_property = core->get_property(target_device, property_item.first));
}
}
} // namespace behavior
} // namespace test
} // namespace ov