IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove ov::hint::PerformanceMode::UNDEFINED (#21592)

Browse Source 
* Remove ov::hint::PerformanceMode::UNDEFINED

* Update for reviewer comments and build issue

* Fix build error - may be used uninitialized

* Update

---------

Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com>
This commit is contained in:
River Li
2023-12-21 01:15:26 +08:00
committed by GitHub
parent 3d3bb51de9
commit dc64268564
8 changed files with 35 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/articles_en/openvino_workflow/running_inference_with_openvino/dldt_deployment_optimization_guide/performance_hints.rst
View File

@@ -23,7 +23,7 @@ Performance Hints: Latency and Throughput
As discussed in the :doc:`Optimization Guide <openvino_docs_deployment_optimization_guide_dldt_optimization_guide>` there are a few different metrics associated with inference speed. Throughput and latency are some of the most widely used metrics that measure the overall performance of an application.
Therefore, in order to ease the configuration of the device, OpenVINO offers two dedicated hints, namely ``ov::hint::PerformanceMode::THROUGHPUT`` and ``ov::hint::PerformanceMode::LATENCY``. A special ``ov::hint::PerformanceMode::UNDEFINED`` hint acts the same as specifying no hint.
Therefore, in order to ease the configuration of the device, OpenVINO offers two dedicated hints, namely ``ov::hint::PerformanceMode::THROUGHPUT`` and ``ov::hint::PerformanceMode::LATENCY``.
For more information on conducting performance measurements with the ``benchmark_app``, refer to the last section in this document.

2
docs/notebooks/108-gpu-device-with-output.rst
View File

@@ -246,7 +246,7 @@ for that property.
GPU_QUEUE_THROTTLE : Priority.MEDIUM
GPU_ENABLE_LOOP_UNROLLING : True
CACHE_DIR :
PERFORMANCE_HINT : PerformanceMode.UNDEFINED
PERFORMANCE_HINT : PerformanceMode.LATENCY
COMPILATION_NUM_THREADS : 20
NUM_STREAMS : 1
PERFORMANCE_HINT_NUM_REQUESTS : 0

57
samples/cpp/benchmark_app/main.cpp
View File

@@ -125,43 +125,52 @@ void next_step(const std::string additional_info = "") {
<< (additional_info.empty() ? "" : " (" + additional_info + ")") << std::endl;
}
ov::hint::PerformanceMode get_performance_hint(const std::string& device, const ov::Core& core) {
OPENVINO_SUPPRESS_DEPRECATED_START
ov::hint::PerformanceMode ov_perf_hint = ov::hint::PerformanceMode::UNDEFINED;
OPENVINO_SUPPRESS_DEPRECATED_END
void handle_performance_hint(const std::string& device, const ov::Core& core, ov::AnyMap& config) {
ov::hint::PerformanceMode ov_perf_hint = ov::hint::PerformanceMode::THROUGHPUT;
auto supported_properties = core.get_property(device, ov::supported_properties);
if (std::find(supported_properties.begin(), supported_properties.end(), ov::hint::performance_mode) !=
supported_properties.end()) {
if (FLAGS_hint != "") {
// Use FLAGS_hint to decide performance mode:
//
// "throughput" or "tput": THROUGHPUT mode
// "cumulative_throughput" or "ctput": CUMULATIVE_THROUGHPUT mode
// "latency": LATENCY mode
// "none": not set ov::hint::performance_mode, let plugin use its default performance mode
// "" : use default THROUGHPUT mode, if FLAG_api="sync" then set LATENCY mode
if (FLAGS_hint != "" && FLAGS_hint != "none") {
if (FLAGS_hint == "throughput" || FLAGS_hint == "tput") {
ov_perf_hint = ov::hint::PerformanceMode::THROUGHPUT;
} else if (FLAGS_hint == "latency") {
ov_perf_hint = ov::hint::PerformanceMode::LATENCY;
} else if (FLAGS_hint == "cumulative_throughput" || FLAGS_hint == "ctput") {
ov_perf_hint = ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT;
} else if (FLAGS_hint == "none") {
OPENVINO_SUPPRESS_DEPRECATED_START
ov_perf_hint = ov::hint::PerformanceMode::UNDEFINED;
OPENVINO_SUPPRESS_DEPRECATED_END
} else {
throw std::logic_error(
"Incorrect performance hint. Please set -hint option to"
"`throughput`(tput), `latency', 'cumulative_throughput'(ctput) value or 'none'.");
}
} else {
ov_perf_hint =
FLAGS_api == "async" ? ov::hint::PerformanceMode::THROUGHPUT : ov::hint::PerformanceMode::LATENCY;
} else if (FLAGS_hint == "") {
ov_perf_hint = ov::hint::PerformanceMode::THROUGHPUT;
if (FLAGS_api == "sync") {
ov_perf_hint = ov::hint::PerformanceMode::LATENCY;
}
slog::warn << "Performance hint was not explicitly specified in command line. "
"Device("
<< device << ") performance hint will be set to " << ov_perf_hint << "." << slog::endl;
}
if (FLAGS_hint != "none") {
// apply command line hint setting and override if hint exists
config[ov::hint::performance_mode.name()] = ov_perf_hint;
} else {
config.erase(ov::hint::performance_mode.name());
}
} else {
if (FLAGS_hint != "") {
if (FLAGS_hint != "none" || FLAGS_hint != "") {
slog::warn << "Device(" << device << ") does not support performance hint property(-hint)." << slog::endl;
}
}
return ov_perf_hint;
return;
}
void setDeviceProperty(ov::Core& core,
@@ -367,20 +376,7 @@ int main(int argc, char* argv[]) {
// Update config per device according to command line parameters
for (auto& device : devices) {
auto& device_config = config[device];
auto ov_perf_hint = get_performance_hint(device, core);
OPENVINO_SUPPRESS_DEPRECATED_START
if (isFlagSetInCommandLine("hint")) {
if (ov_perf_hint != ov::hint::PerformanceMode::UNDEFINED) {
// apply command line hint setting and override if hint exists
device_config[ov::hint::performance_mode.name()] = ov_perf_hint;
} else {
device_config.erase(ov::hint::performance_mode.name());
}
} else if (ov_perf_hint != ov::hint::PerformanceMode::UNDEFINED) {
// keep hint setting in the config if no hint setting from command line
device_config.emplace(ov::hint::performance_mode(ov_perf_hint));
}
OPENVINO_SUPPRESS_DEPRECATED_END
handle_performance_hint(device, core, device_config);
if (FLAGS_nireq != 0)
device_config[ov::hint::num_requests.name()] = unsigned(FLAGS_nireq);
@@ -443,8 +439,7 @@ int main(int argc, char* argv[]) {
"<dev1>:<nstreams1>,<dev2>:<nstreams2>" +
" or via configuration file.");
}
} else if (ov_perf_hint == ov::hint::PerformanceMode::UNDEFINED && !device_config.count(key) &&
(FLAGS_api == "async")) {
} else if (FLAGS_api == "none" && !device_config.count(key) && (FLAGS_api == "async")) {
slog::warn << "-nstreams default value is determined automatically for " << device
<< " device. "
"Although the automatic selection usually provides a "

1
src/bindings/python/src/pyopenvino/core/properties/properties.cpp
View File

@@ -57,7 +57,6 @@ void regmodule_properties(py::module m) {
OPENVINO_SUPPRESS_DEPRECATED_START
py::enum_<ov::hint::PerformanceMode>(m_hint, "PerformanceMode", py::arithmetic())
.value("UNDEFINED", ov::hint::PerformanceMode::UNDEFINED)
.value("LATENCY", ov::hint::PerformanceMode::LATENCY)
.value("THROUGHPUT", ov::hint::PerformanceMode::THROUGHPUT)
.value("CUMULATIVE_THROUGHPUT", ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT);

10
src/bindings/python/tests/test_runtime/test_properties.py
View File

@@ -37,13 +37,6 @@ def test_properties_rw_base():
assert "incompatible function arguments" in str(e.value)
def test_deprecation():
with pytest.warns(DeprecationWarning) as w:
_ = hints.PerformanceMode.UNDEFINED
assert issubclass(w[0].category, DeprecationWarning)
assert "PerformanceMode.UNDEFINED is deprecated and will be removed" in str(w[0].message)
###
# Enum-like values
###
@@ -71,7 +64,6 @@ def test_deprecation():
(
hints.PerformanceMode,
(
(hints.PerformanceMode.UNDEFINED, "PerformanceMode.UNDEFINED", -1),
(hints.PerformanceMode.LATENCY, "PerformanceMode.LATENCY", 1),
(hints.PerformanceMode.THROUGHPUT, "PerformanceMode.THROUGHPUT", 2),
(hints.PerformanceMode.CUMULATIVE_THROUGHPUT, "PerformanceMode.CUMULATIVE_THROUGHPUT", 3),
@@ -253,7 +245,7 @@ def test_properties_ro(ov_property_ro, expected_value):
(
hints.performance_mode,
"PERFORMANCE_HINT",
((hints.PerformanceMode.UNDEFINED, hints.PerformanceMode.UNDEFINED),),
((hints.PerformanceMode.THROUGHPUT, hints.PerformanceMode.THROUGHPUT),),
),
(
hints.enable_cpu_pinning,

10
src/inference/include/openvino/runtime/properties.hpp
View File

@@ -300,8 +300,6 @@ static constexpr Property<Priority> model_priority{"MODEL_PRIORITY"};
* @ingroup ov_runtime_cpp_prop_api
*/
enum class PerformanceMode {
UNDEFINED OPENVINO_ENUM_DEPRECATED("Please use actual value instead. Will be removed in 2024.0") =
-1, //!< Undefined value, performance setting may vary from device to device
LATENCY = 1, //!< Optimize for latency
THROUGHPUT = 2, //!< Optimize for throughput
CUMULATIVE_THROUGHPUT = 3, //!< Optimize for cumulative throughput
@@ -310,10 +308,6 @@ enum class PerformanceMode {
/** @cond INTERNAL */
inline std::ostream& operator<<(std::ostream& os, const PerformanceMode& performance_mode) {
switch (performance_mode) {
OPENVINO_SUPPRESS_DEPRECATED_START
case PerformanceMode::UNDEFINED:
return os << "UNDEFINED";
OPENVINO_SUPPRESS_DEPRECATED_END
case PerformanceMode::LATENCY:
return os << "LATENCY";
case PerformanceMode::THROUGHPUT:
@@ -334,10 +328,6 @@ inline std::istream& operator>>(std::istream& is, PerformanceMode& performance_m
performance_mode = PerformanceMode::THROUGHPUT;
} else if (str == "CUMULATIVE_THROUGHPUT") {
performance_mode = PerformanceMode::CUMULATIVE_THROUGHPUT;
} else if (str == "UNDEFINED") {
OPENVINO_SUPPRESS_DEPRECATED_START
performance_mode = PerformanceMode::UNDEFINED;
OPENVINO_SUPPRESS_DEPRECATED_END
} else {
OPENVINO_THROW("Unsupported performance mode: ", str);
}

3
src/plugins/intel_gpu/src/runtime/execution_config.cpp
View File

@@ -30,8 +30,7 @@ public:
OPENVINO_SUPPRESS_DEPRECATED_START
return mode == ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT ||
mode == ov::hint::PerformanceMode::THROUGHPUT ||
mode == ov::hint::PerformanceMode::LATENCY ||
mode == ov::hint::PerformanceMode::UNDEFINED;
mode == ov::hint::PerformanceMode::LATENCY;
OPENVINO_SUPPRESS_DEPRECATED_END
}
};

10
tools/benchmark_tool/openvino/tools/benchmark/main.py
View File

@@ -106,7 +106,7 @@ def main():
next_step()
def set_performance_hint(device):
perf_hint = properties.hint.PerformanceMode.UNDEFINED
perf_hint = properties.hint.PerformanceMode.THROUGHPUT
supported_properties = benchmark.core.get_property(device, properties.supported_properties())
if properties.hint.performance_mode() in supported_properties:
if is_flag_set_in_command_line('hint'):
@@ -117,16 +117,16 @@ def main():
elif args.perf_hint == "cumulative_throughput" or args.perf_hint == "ctput":
perf_hint = properties.hint.PerformanceMode.CUMULATIVE_THROUGHPUT
elif args.perf_hint=='none':
perf_hint = properties.hint.PerformanceMode.UNDEFINED
# Not set PerformanceMode, and plugin will apply its internal default PerformanceMode
return
else:
raise RuntimeError("Incorrect performance hint. Please set -hint option to"
"`throughput`(tput), `latency', 'cumulative_throughput'(ctput) value or 'none'.")
else:
perf_hint = properties.hint.PerformanceMode.THROUGHPUT if benchmark.api_type == "async" else properties.hint.PerformanceMode.LATENCY
perf_hint = properties.hint.PerformanceMode.LATENCY if benchmark.api_type == "sync" else properties.hint.PerformanceMode.THROUGHPUT
logger.warning(f"Performance hint was not explicitly specified in command line. " +
f"Device({device}) performance hint will be set to {perf_hint}.")
if perf_hint != properties.hint.PerformanceMode.UNDEFINED:
config[device][properties.hint.performance_mode()] = perf_hint
config[device][properties.hint.performance_mode()] = perf_hint
else:
logger.warning(f"Device {device} does not support performance hint property(-hint).")