From 815d4abc03dacff415758dc5fcc6e55c60d33756 Mon Sep 17 00:00:00 2001 From: "Shen, Wanglei" Date: Tue, 28 Mar 2023 04:39:26 +0800 Subject: [PATCH] enable new property ov::hint::use_hyper_threading (#16176) * enable apply_processor_type() * declare PROCESSOR_TYPE * enable readProperties * test case for get_property() * enable set_property() and test cases * reduce changes * fix code style issue * fix python test case issue * remove python interface * move processor type definition out of dev_api * refine coding * add dependency * update header file * update description * merge intel_cpu header file * add inline in-code documentation * change 'UNDEFINED' to 'DEFAULT' * remove ProcTypeConfig * refine change * refine change * enable new property use hyper threading * update description * resume legacy code * change to ov::hint namespace * update including header file * update C API and Python API * update description for comments * update test case for comments * update function location for comments * fix typo * fix typo * fix code style issue and update test case * move cpu_map_scheduling into threading folder --- .../c/include/openvino/c/ov_property.h | 7 ++ src/bindings/c/src/ov_property.cpp | 1 + src/bindings/c/tests/ov_core_test.cpp | 24 +++++ .../pyopenvino/core/properties/properties.cpp | 1 + .../tests/test_runtime/test_properties.py | 12 +++ .../runtime/threading/cpu_map_scheduling.hpp | 29 ++++++ .../include/openvino/runtime/properties.hpp | 16 ++++ .../src/dev/threading/cpu_map_scheduling.cpp | 28 ++++++ src/plugins/intel_cpu/src/config.cpp | 11 +++ src/plugins/intel_cpu/src/config.h | 2 + .../intel_cpu/src/cpu_streams_calculation.hpp | 3 +- src/plugins/intel_cpu/src/exec_network.cpp | 4 + src/plugins/intel_cpu/src/plugin.cpp | 6 +- .../behavior/ov_plugin/core_integration.cpp | 4 + .../tests/unit/streams_info_table_test.cpp | 91 +++++++++++++++++++ .../behavior/ov_plugin/core_integration.hpp | 18 ++++ 16 files changed, 255 insertions(+), 2 deletions(-) create mode 100644 src/inference/dev_api/openvino/runtime/threading/cpu_map_scheduling.hpp create mode 100644 src/inference/src/dev/threading/cpu_map_scheduling.cpp diff --git a/src/bindings/c/include/openvino/c/ov_property.h b/src/bindings/c/include/openvino/c/ov_property.h index 54c887435c5..63146c7f48e 100644 --- a/src/bindings/c/include/openvino/c/ov_property.h +++ b/src/bindings/c/include/openvino/c/ov_property.h @@ -119,6 +119,13 @@ ov_property_key_affinity; OPENVINO_C_VAR(const char*) ov_property_key_inference_num_threads; +/** + * @brief Read-write property, it is high-level OpenVINO hint for using hyper threading processors during CPU inference + * @ingroup ov_property_c_api + */ +OPENVINO_C_VAR(const char*) +ov_property_key_hint_use_hyper_threading; + /** * @brief Read-write property, it is high-level OpenVINO Performance Hints * @ingroup ov_property_c_api diff --git a/src/bindings/c/src/ov_property.cpp b/src/bindings/c/src/ov_property.cpp index 2d6c470ae5d..1fe6e9397e2 100644 --- a/src/bindings/c/src/ov_property.cpp +++ b/src/bindings/c/src/ov_property.cpp @@ -23,6 +23,7 @@ const char* ov_property_key_num_streams = "NUM_STREAMS"; const char* ov_property_key_affinity = "AFFINITY"; const char* ov_property_key_inference_num_threads = "INFERENCE_NUM_THREADS"; const char* ov_property_key_hint_performance_mode = "PERFORMANCE_HINT"; +const char* ov_property_key_hint_use_hyper_threading = "USE_HYPER_THREADING"; const char* ov_property_key_hint_inference_precision = "INFERENCE_PRECISION_HINT"; const char* ov_property_key_hint_num_requests = "PERFORMANCE_HINT_NUM_REQUESTS"; const char* ov_property_key_hint_model_priority = "MODEL_PRIORITY"; diff --git a/src/bindings/c/tests/ov_core_test.cpp b/src/bindings/c/tests/ov_core_test.cpp index 0cb2f29f65e..8fe3f41d9cc 100644 --- a/src/bindings/c/tests/ov_core_test.cpp +++ b/src/bindings/c/tests/ov_core_test.cpp @@ -259,6 +259,22 @@ TEST_P(ov_core_test, ov_core_set_property_enum_invalid) { OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key, &ret)); EXPECT_STRNE(invalid_mode, ret); ov_free(ret); + + const char* key_ht = ov_property_key_hint_use_hyper_threading; + const char* val_ht = "YES"; + OV_EXPECT_OK(ov_core_set_property(core, device_name.c_str(), key_ht, val_ht)); + ret = nullptr; + OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key_ht, &ret)); + EXPECT_STREQ(val_ht, ret); + ov_free(ret); + + const char* invalid_val = "INVALID_VAL"; + OV_EXPECT_NOT_OK(ov_core_set_property(core, device_name.c_str(), key_ht, invalid_val)); + ret = nullptr; + OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key_ht, &ret)); + EXPECT_STRNE(invalid_val, ret); + ov_free(ret); + ov_core_free(core); } @@ -276,6 +292,14 @@ TEST_P(ov_core_test, ov_core_set_and_get_property_enum) { EXPECT_STREQ(affinity, ret); ov_free(ret); + const char* key_ht = ov_property_key_hint_use_hyper_threading; + const char* val_ht = "YES"; + OV_EXPECT_OK(ov_core_set_property(core, device_name.c_str(), key_ht, val_ht)); + ret = nullptr; + OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key_ht, &ret)); + EXPECT_STREQ(val_ht, ret); + ov_free(ret); + ov_core_free(core); } diff --git a/src/bindings/python/src/pyopenvino/core/properties/properties.cpp b/src/bindings/python/src/pyopenvino/core/properties/properties.cpp index 7c45761700f..da9e4e0af51 100644 --- a/src/bindings/python/src/pyopenvino/core/properties/properties.cpp +++ b/src/bindings/python/src/pyopenvino/core/properties/properties.cpp @@ -67,6 +67,7 @@ void regmodule_properties(py::module m) { wrap_property_RW(m_hint, ov::hint::inference_precision, "inference_precision"); wrap_property_RW(m_hint, ov::hint::model_priority, "model_priority"); wrap_property_RW(m_hint, ov::hint::performance_mode, "performance_mode"); + wrap_property_RW(m_hint, ov::hint::use_hyper_threading, "use_hyper_threading"); wrap_property_RW(m_hint, ov::hint::execution_mode, "execution_mode"); wrap_property_RW(m_hint, ov::hint::num_requests, "num_requests"); wrap_property_RW(m_hint, ov::hint::model, "model"); diff --git a/src/bindings/python/tests/test_runtime/test_properties.py b/src/bindings/python/tests/test_runtime/test_properties.py index 41558de5aa7..6eecdfb090b 100644 --- a/src/bindings/python/tests/test_runtime/test_properties.py +++ b/src/bindings/python/tests/test_runtime/test_properties.py @@ -219,6 +219,16 @@ def test_properties_ro(ov_property_ro, expected_value): "PERFORMANCE_HINT", ((properties.hint.PerformanceMode.UNDEFINED, properties.hint.PerformanceMode.UNDEFINED),), ), + ( + properties.hint.use_hyper_threading, + "USE_HYPER_THREADING", + ( + (True, True), + (False, False), + (1, True), + (0, False), + ), + ), ( properties.hint.execution_mode, "EXECUTION_MODE_HINT", @@ -399,6 +409,7 @@ def test_single_property_setting(device): properties.affinity(properties.Affinity.NONE), properties.inference_precision(Type.f32), properties.hint.performance_mode(properties.hint.PerformanceMode.LATENCY), + properties.hint.use_hyper_threading(True), properties.hint.num_requests(12), properties.streams.num(5), ], @@ -411,6 +422,7 @@ def test_single_property_setting(device): properties.affinity(): properties.Affinity.NONE, properties.inference_precision(): Type.f32, properties.hint.performance_mode(): properties.hint.PerformanceMode.LATENCY, + properties.hint.use_hyper_threading(): True, properties.hint.num_requests(): 12, properties.streams.num(): 5, }, diff --git a/src/inference/dev_api/openvino/runtime/threading/cpu_map_scheduling.hpp b/src/inference/dev_api/openvino/runtime/threading/cpu_map_scheduling.hpp new file mode 100644 index 00000000000..d00929bfe24 --- /dev/null +++ b/src/inference/dev_api/openvino/runtime/threading/cpu_map_scheduling.hpp @@ -0,0 +1,29 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief A header file for CPU map scheduling + * @file cpu_map_scheduling.hpp + */ + +#pragma once + +#include + +#include "openvino/runtime/properties.hpp" + +namespace ov { + +/** + * @brief Limit available CPU resource in processors type table according to hyper threading property + * @param[in] input_type indicate value of property use_hyper_threading. + * @param[in] input_changed indicate if value is set by user. + * @param[in] proc_type_table candidate processors available at this time + * @return updated proc_type_table which removed unmatched processors + */ +std::vector> apply_hyper_threading(bool input_type, + const bool input_changed, + const std::vector> proc_type_table); + +} // namespace ov \ No newline at end of file diff --git a/src/inference/include/openvino/runtime/properties.hpp b/src/inference/include/openvino/runtime/properties.hpp index d5a1853cb5f..5b567394144 100644 --- a/src/inference/include/openvino/runtime/properties.hpp +++ b/src/inference/include/openvino/runtime/properties.hpp @@ -351,6 +351,22 @@ inline std::istream& operator>>(std::istream& is, PerformanceMode& performance_m */ static constexpr Property performance_mode{"PERFORMANCE_HINT"}; +/** + * @brief This property allows hyper threading during inference. + * @ingroup ov_runtime_cpp_prop_api + * + * Developer can use this property to use or not use hyper threading during inference. If user does not explicitly set + * value for this property, OpenVINO may choose any desired value based on internal logic. + * + * The following code is example to use this property. + * + * @code + * ie.set_property(ov::hint::use_hyper_threading(true)); + * ie.set_property(ov::hint::use_hyper_threading(false)); + * @endcode + */ +static constexpr Property use_hyper_threading{"USE_HYPER_THREADING"}; + /** * @brief (Optional) property that backs the (above) Performance Hints * by giving additional information on how many inference requests the application will be keeping in flight diff --git a/src/inference/src/dev/threading/cpu_map_scheduling.cpp b/src/inference/src/dev/threading/cpu_map_scheduling.cpp new file mode 100644 index 00000000000..42ed0e33272 --- /dev/null +++ b/src/inference/src/dev/threading/cpu_map_scheduling.cpp @@ -0,0 +1,28 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/runtime/threading/cpu_map_scheduling.hpp" + +#include "ie_system_conf.h" + +namespace ov { + +std::vector> apply_hyper_threading(bool input_value, + const bool input_changed, + const std::vector> proc_type_table) { + std::vector> result_table = proc_type_table; + + if ((proc_type_table[0][HYPER_THREADING_PROC] > 0) && + (((!input_value) && input_changed) || ((!input_changed) && (proc_type_table.size() > 1)))) { + for (auto& i : result_table) { + i[ALL_PROC] -= i[HYPER_THREADING_PROC]; + i[HYPER_THREADING_PROC] = 0; + } + input_value = false; + } + + return result_table; +} + +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/config.cpp b/src/plugins/intel_cpu/src/config.cpp index 04c8c646768..be8a40d5626 100644 --- a/src/plugins/intel_cpu/src/config.cpp +++ b/src/plugins/intel_cpu/src/config.cpp @@ -78,6 +78,17 @@ void Config::readProperties(const std::map &prop) { streamExecutorConfig.SetConfig(key, val); } else if (hintsConfigKeys.end() != std::find(hintsConfigKeys.begin(), hintsConfigKeys.end(), key)) { perfHintsConfig.SetConfig(key, val); + } else if (key == ov::hint::use_hyper_threading.name()) { + if (val == PluginConfigParams::YES) { + useHyperThreading = true; + changedHyperThreading = true; + } else if (val == PluginConfigParams::NO) { + useHyperThreading = false; + changedHyperThreading = true; + } else { + IE_THROW() << "Wrong value " << val << "for property key " << ov::hint::use_hyper_threading.name() + << ". Expected only true/false." << std::endl; + } } else if (key == PluginConfigParams::KEY_DYN_BATCH_LIMIT) { int val_i = -1; try { diff --git a/src/plugins/intel_cpu/src/config.h b/src/plugins/intel_cpu/src/config.h index 8c399d5189a..27f83229002 100644 --- a/src/plugins/intel_cpu/src/config.h +++ b/src/plugins/intel_cpu/src/config.h @@ -49,6 +49,8 @@ struct Config { size_t rtCacheCapacity = 5000ul; InferenceEngine::IStreamsExecutor::Config streamExecutorConfig; InferenceEngine::PerfHintsConfig perfHintsConfig; + bool useHyperThreading = true; + bool changedHyperThreading = false; #if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) LPTransformsMode lpTransformsMode = LPTransformsMode::On; bool enforceBF16 = true; diff --git a/src/plugins/intel_cpu/src/cpu_streams_calculation.hpp b/src/plugins/intel_cpu/src/cpu_streams_calculation.hpp index 53614942bf8..e331035567b 100644 --- a/src/plugins/intel_cpu/src/cpu_streams_calculation.hpp +++ b/src/plugins/intel_cpu/src/cpu_streams_calculation.hpp @@ -13,6 +13,7 @@ namespace ov { namespace intel_cpu { + /** * @brief Generate streams information table according to processors type table. * @param[in] input_streams is the targeted number of streams set by user via ov::num_streams or hints. @@ -41,4 +42,4 @@ std::vector> get_streams_info_table(const int input_streams, const int model_prefer_threads, const std::vector> proc_type_table); } // namespace intel_cpu -} // namespace ov +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/src/exec_network.cpp b/src/plugins/intel_cpu/src/exec_network.cpp index 0395c7a416e..72dc81f58f8 100644 --- a/src/plugins/intel_cpu/src/exec_network.cpp +++ b/src/plugins/intel_cpu/src/exec_network.cpp @@ -308,6 +308,7 @@ InferenceEngine::Parameter ExecNetwork::GetMetric(const std::string &name) const RO_property(ov::inference_precision.name()), RO_property(ov::hint::performance_mode.name()), RO_property(ov::hint::num_requests.name()), + RO_property(ov::hint::use_hyper_threading.name()), RO_property(ov::execution_devices.name()), }; } @@ -348,6 +349,9 @@ InferenceEngine::Parameter ExecNetwork::GetMetric(const std::string &name) const } else if (name == ov::hint::performance_mode) { const auto perfHint = ov::util::from_string(config.perfHintsConfig.ovPerfHint, ov::hint::performance_mode); return perfHint; + } else if (name == ov::hint::use_hyper_threading.name()) { + const bool use_ht = config.useHyperThreading; + return decltype(ov::hint::use_hyper_threading)::value_type(use_ht); } else if (name == ov::hint::num_requests) { const auto perfHintNumRequests = config.perfHintsConfig.ovPerfHintNumRequests; return decltype(ov::hint::num_requests)::value_type(perfHintNumRequests); diff --git a/src/plugins/intel_cpu/src/plugin.cpp b/src/plugins/intel_cpu/src/plugin.cpp index 7aba2c8e0f6..b2af5acfea3 100644 --- a/src/plugins/intel_cpu/src/plugin.cpp +++ b/src/plugins/intel_cpu/src/plugin.cpp @@ -21,7 +21,7 @@ #include #include "performance_heuristics.hpp" - +#include "openvino/runtime/properties.hpp" #include "weights_cache.hpp" #include "utils/denormals.hpp" @@ -514,6 +514,9 @@ Parameter Engine::GetConfig(const std::string& name, const std::map #include +#include #include "cpu_streams_calculation.hpp" @@ -15,6 +16,96 @@ using namespace ov; namespace { +struct UseHTTestCase { + bool use_ht_value; + bool use_ht_changed; + std::vector> proc_type_table; + std::vector> result_table; +}; + +class UseHTTests : public CommonTestUtils::TestsCommon, public testing::WithParamInterface> { +public: + void SetUp() override { + const auto& test_data = std::get<0>(GetParam()); + + std::vector> test_result_table = + ov::apply_hyper_threading(test_data.use_ht_value, + test_data.use_ht_changed, + test_data.proc_type_table); + + ASSERT_EQ(test_data.result_table, test_result_table); + } +}; + +UseHTTestCase _2sockets_false = { + false, + true, + {{208, 104, 0, 104}, {104, 52, 0, 52}, {104, 52, 0, 52}}, + {{104, 104, 0, 0}, {52, 52, 0, 0}, {52, 52, 0, 0}}, +}; + +UseHTTestCase _2sockets_true = { + true, + true, + {{208, 104, 0, 104}, {104, 52, 0, 52}, {104, 52, 0, 52}}, + {{208, 104, 0, 104}, {104, 52, 0, 52}, {104, 52, 0, 52}}, +}; + +UseHTTestCase _2sockets_default_1 = { + false, + false, + {{208, 104, 0, 104}, {104, 52, 0, 52}, {104, 52, 0, 52}}, + {{104, 104, 0, 0}, {52, 52, 0, 0}, {52, 52, 0, 0}}, +}; + +UseHTTestCase _2sockets_default_2 = { + true, + false, + {{208, 104, 0, 104}, {104, 52, 0, 52}, {104, 52, 0, 52}}, + {{104, 104, 0, 0}, {52, 52, 0, 0}, {52, 52, 0, 0}}, +}; + +UseHTTestCase _1sockets_false = { + false, + true, + {{20, 6, 8, 6}}, + {{14, 6, 8, 0}}, +}; + +UseHTTestCase _1sockets_true = { + true, + true, + {{20, 6, 8, 6}}, + {{20, 6, 8, 6}}, +}; + +UseHTTestCase _1sockets_default_1 = { + false, + false, + {{20, 6, 8, 6}}, + {{20, 6, 8, 6}}, +}; + +UseHTTestCase _1sockets_default_2 = { + true, + false, + {{20, 6, 8, 6}}, + {{20, 6, 8, 6}}, +}; + +TEST_P(UseHTTests, UseHT) {} + +INSTANTIATE_TEST_SUITE_P(UseHTTable, + UseHTTests, + testing::Values(_2sockets_false, + _2sockets_true, + _2sockets_default_1, + _2sockets_default_2, + _1sockets_false, + _1sockets_true, + _1sockets_default_1, + _1sockets_default_2)); + struct StreamsCalculationTestCase { int input_streams; int input_threads; diff --git a/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp index e5421ea7b8c..fa9b3f22fcb 100644 --- a/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp +++ b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp @@ -121,6 +121,7 @@ using OVClassLoadNetworkTest = OVClassQueryNetworkTest; using OVClassSetGlobalConfigTest = OVClassBaseTestP; using OVClassSetModelPriorityConfigTest = OVClassBaseTestP; using OVClassSetExecutionModeHintConfigTest = OVClassBaseTestP; +using OVClassSetUseHyperThreadingHintConfigTest = OVClassBaseTestP; using OVClassSetTBBForceTerminatePropertyTest = OVClassBaseTestP; using OVClassSetLogLevelConfigTest = OVClassBaseTestP; using OVClassSpecificDeviceTestSetConfig = OVClassBaseTestP; @@ -611,6 +612,23 @@ TEST_P(OVClassSetExecutionModeHintConfigTest, SetConfigNoThrow) { ASSERT_EQ(ov::hint::ExecutionMode::PERFORMANCE, ie.get_property(target_device, ov::hint::execution_mode)); } +TEST_P(OVClassSetUseHyperThreadingHintConfigTest, SetConfigNoThrow) { + ov::Core ie = createCoreWithTemplate(); + + OV_ASSERT_PROPERTY_SUPPORTED(ov::hint::use_hyper_threading); + + bool defaultMode{}; + ASSERT_NO_THROW(defaultMode = ie.get_property(target_device, ov::hint::use_hyper_threading)); + (void)defaultMode; + + ASSERT_EQ(true, ie.get_property(target_device, ov::hint::use_hyper_threading)); + + ie.set_property(target_device, ov::hint::use_hyper_threading(false)); + ASSERT_EQ(false, ie.get_property(target_device, ov::hint::use_hyper_threading)); + ie.set_property(target_device, ov::hint::use_hyper_threading(true)); + ASSERT_EQ(true, ie.get_property(target_device, ov::hint::use_hyper_threading)); +} + TEST_P(OVClassSetDevicePriorityConfigTest, SetConfigAndCheckGetConfigNoThrow) { ov::Core ie = createCoreWithTemplate(); std::string devicePriority;