From 9dec8db96490aea4954ddbd628f58b038af68759 Mon Sep 17 00:00:00 2001 From: Anton Pankratov Date: Wed, 23 Feb 2022 13:03:37 +0300 Subject: [PATCH] Common OV configuration tests (#10286) * Used new config for streams and threads * Fixed review coments in ba * format fix * fixed hello_query_device * Added STL string io * fixed tests * Fixed test * Fixed build * fixed format * Fixed build * try fix win * other any io specialization * Fixed after merge * renamed streams * build fixed * fixed build * fixed format * fix for old mac build * Fixed type of exception * test fix * Added ov configuration test * Added common OV properties tests * fix mklnn * fixed foramat * merge conflicts * Remoed compile_model tests * removed duplicated test --- .../behavior/ov_plugin/properties_tests.cpp | 120 ++++++++++++++++++ .../include/openvino/runtime/properties.hpp | 10 +- .../behavior/ov_plugin/properties_tests.hpp | 52 ++++++++ .../behavior/ov_plugin/properties_tests.cpp | 101 +++++++++++++++ 4 files changed, 278 insertions(+), 5 deletions(-) create mode 100644 docs/template_plugin/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp create mode 100644 src/tests/functional/plugin/shared/include/behavior/ov_plugin/properties_tests.hpp create mode 100644 src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp diff --git a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp new file mode 100644 index 00000000000..d6a87759e01 --- /dev/null +++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp @@ -0,0 +1,120 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_plugin/properties_tests.hpp" +#include "openvino/runtime/properties.hpp" + +using namespace ov::test::behavior; + +namespace { + +const std::vector inproperties = { + {ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")}, +}; + +const std::vector hetero_inproperties = { + {ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")}, +}; + +const std::vector multi_inproperties = { + {ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")}, +}; + + +const std::vector auto_inproperties = { + {ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")}, +}; + + +const std::vector auto_batch_inproperties = { + {ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")}, +}; + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_BehaviorTests, OVPropertiesIncorrectTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE), + ::testing::ValuesIn(inproperties)), + OVPropertiesIncorrectTests::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_Hetero_BehaviorTests, OVPropertiesIncorrectTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_HETERO), + ::testing::ValuesIn(hetero_inproperties)), + OVPropertiesIncorrectTests::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_Multi_BehaviorTests, OVPropertiesIncorrectTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_MULTI), + ::testing::ValuesIn(multi_inproperties)), + OVPropertiesIncorrectTests::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_Auto_BehaviorTests, OVPropertiesIncorrectTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_AUTO), + ::testing::ValuesIn(auto_inproperties)), + OVPropertiesIncorrectTests::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_AutoBatch_BehaviorTests, OVPropertiesIncorrectTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_BATCH), + ::testing::ValuesIn(auto_batch_inproperties)), + OVPropertiesIncorrectTests::getTestCaseName); + +const std::vector default_properties = { + {ov::enable_profiling(true)}, + {ov::device::id(0)}, +}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVPropertiesDefaultTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE), + ::testing::ValuesIn(default_properties)), + OVPropertiesDefaultTests::getTestCaseName); + +const std::vector properties = { + {ov::enable_profiling(true)}, + {ov::device::id(0)}, +}; + +const std::vector hetero_properties = { + {ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::enable_profiling(true)}, + {ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::device::id(0)}, +}; + + +const std::vector multi_properties = { + {ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::enable_profiling(true)}, + {ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::device::id(0)}, +}; + +const std::vector auto_batch_properties = { + {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , CommonTestUtils::DEVICE_TEMPLATE}}, + {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , CommonTestUtils::DEVICE_TEMPLATE}, + {CONFIG_KEY(AUTO_BATCH_TIMEOUT) , "1"}}, +}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVPropertiesTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE), + ::testing::ValuesIn(properties)), + OVPropertiesTests::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_Hetero_BehaviorTests, OVPropertiesTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_HETERO), + ::testing::ValuesIn(hetero_properties)), + OVPropertiesTests::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_Multi_BehaviorTests, OVPropertiesTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_MULTI), + ::testing::ValuesIn(multi_properties)), + OVPropertiesTests::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_AutoBatch_BehaviorTests, OVPropertiesTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_BATCH), + ::testing::ValuesIn(auto_batch_properties)), + OVPropertiesTests::getTestCaseName); +} // namespace diff --git a/src/inference/include/openvino/runtime/properties.hpp b/src/inference/include/openvino/runtime/properties.hpp index b3935c0f96e..1c8632db316 100644 --- a/src/inference/include/openvino/runtime/properties.hpp +++ b/src/inference/include/openvino/runtime/properties.hpp @@ -138,7 +138,7 @@ class Property : public util::BaseProperty { template struct Forward { template ::value && + typename std::enable_if::type, std::string>::value && std::is_convertible::value, bool>::type = true> explicit operator U() { @@ -146,15 +146,15 @@ class Property : public util::BaseProperty { } template ::value && + typename std::enable_if::type, std::string>::value && !std::is_convertible::value, bool>::type = true> explicit operator U() { - return Any{value}.as(); + return Any{value}.as(); } template ::value && + typename std::enable_if::type, std::string>::value && std::is_convertible::value, bool>::type = true> explicit operator U() { @@ -162,7 +162,7 @@ class Property : public util::BaseProperty { } template ::value && + typename std::enable_if::type, std::string>::value && !std::is_convertible::value, bool>::type = true> explicit operator U() { diff --git a/src/tests/functional/plugin/shared/include/behavior/ov_plugin/properties_tests.hpp b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/properties_tests.hpp new file mode 100644 index 00000000000..3f90e4b68ca --- /dev/null +++ b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/properties_tests.hpp @@ -0,0 +1,52 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "base/ov_behavior_test_utils.hpp" +#include "openvino/runtime/properties.hpp" +#include "openvino/util/common_util.hpp" + +#include "common_test_utils/test_assertions.hpp" +#include "common_test_utils/file_utils.hpp" +#include "common_test_utils/unicode_utils.hpp" + +namespace ov { +namespace test { +namespace behavior { + +class OVPropertiesBase : public CommonTestUtils::TestsCommon { +public: + std::shared_ptr core = utils::PluginCache::get().core(); + std::shared_ptr model; + std::string device_name; + AnyMap properties; +}; + +class OVEmptyPropertiesTests : public testing::WithParamInterface, + public OVPropertiesBase { +public: + static std::string getTestCaseName(testing::TestParamInfo obj); + + void SetUp() override; +}; + +using PropertiesParams = std::tuple; + +class OVPropertiesTests : public testing::WithParamInterface, + public OVPropertiesBase { +public: + static std::string getTestCaseName(testing::TestParamInfo obj); + + void SetUp() override; + + void TearDown() override; +}; + +using OVPropertiesIncorrectTests = OVPropertiesTests; +using OVPropertiesDefaultTests = OVPropertiesTests; + +} // namespace behavior +} // namespace test +} // namespace ov \ No newline at end of file diff --git a/src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp b/src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp new file mode 100644 index 00000000000..a7d4083c99a --- /dev/null +++ b/src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp @@ -0,0 +1,101 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_plugin/properties_tests.hpp" +#include "openvino/runtime/properties.hpp" +#include + +namespace ov { +namespace test { +namespace behavior { + +std::string OVEmptyPropertiesTests::getTestCaseName(testing::TestParamInfo obj) { + return "device_name=" + obj.param; +} + +void OVEmptyPropertiesTests::SetUp() { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + device_name = this->GetParam(); + model = ngraph::builder::subgraph::makeConvPoolRelu(); +} + +std::string OVPropertiesTests::getTestCaseName(testing::TestParamInfo obj) { + std::string device_name; + AnyMap properties; + std::tie(device_name, properties) = obj.param; + std::ostringstream result; + result << "device_name=" << device_name << "_"; + if (!properties.empty()) { + result << "properties=" << util::join(util::split(util::to_string(properties), ' '), "_"); + } + return result.str(); +} + +void OVPropertiesTests::SetUp() { + SKIP_IF_CURRENT_TEST_IS_DISABLED(); + std::tie(device_name, properties) = this->GetParam(); + model = ngraph::builder::subgraph::makeConvPoolRelu(); +} + +void OVPropertiesTests::TearDown() { + if (!properties.empty()) { + utils::PluginCache::get().reset(); + } +} + +TEST_P(OVEmptyPropertiesTests, SetEmptyProperties) { + OV_ASSERT_NO_THROW(core->get_property(device_name, ov::supported_properties)); + OV_ASSERT_NO_THROW(core->set_property(device_name, AnyMap{})); +} + +// Setting correct properties doesn't throw +TEST_P(OVPropertiesTests, SetCorrectProperties) { + OV_ASSERT_NO_THROW(core->set_property(device_name, properties)); +} + +TEST_P(OVPropertiesTests, canSetPropertyAndCheckGetProperty) { + core->set_property(device_name, properties); + for (const auto& property_item : properties) { + Any property; + OV_ASSERT_NO_THROW(property = core->get_property(device_name, property_item.first)); + ASSERT_FALSE(property.empty()); + std::cout << property_item.first << ":" << property.as() << std::endl; + } +} + +TEST_P(OVPropertiesIncorrectTests, SetPropertiesWithIncorrectKey) { + ASSERT_THROW(core->set_property(device_name, properties), ov::Exception); +} + +TEST_P(OVPropertiesIncorrectTests, CanNotCompileModelWithIncorrectProperties) { + ASSERT_THROW(core->compile_model(model, device_name, properties), ov::Exception); +} + +TEST_P(OVPropertiesDefaultTests, CanSetDefaultValueBackToPlugin) { + std::vector supported_properties; + OV_ASSERT_NO_THROW(supported_properties = core->get_property(device_name, ov::supported_properties)); + for (auto& supported_property : supported_properties) { + Any property; + OV_ASSERT_NO_THROW(property = core->get_property(device_name, supported_property)); + if (supported_property.is_mutable()) { + OV_ASSERT_NO_THROW(core->set_property(device_name, {{ supported_property, property}})); + } + } +} + +TEST_P(OVPropertiesDefaultTests, CheckDefaultValues) { + std::vector supported_properties; + OV_ASSERT_NO_THROW(supported_properties = core->get_property(device_name, ov::supported_properties)); + for (auto&& default_property : properties) { + auto supported = util::contains(supported_properties, default_property.first); + ASSERT_TRUE(supported) << "default_property=" << default_property.first; + Any property; + OV_ASSERT_NO_THROW(property = core->get_property(device_name, default_property.first)); + ASSERT_EQ(default_property.second, property); + } +} + +} // namespace behavior +} // namespace test +} // namespace ov