IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update the logic of benchmark app property setting (#16427)

Browse Source 
* 1. refine the logic to ov::device::properties setting.
2. the config overrides will be performed if same config setting is came from CMD line.-a

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Update configuration sample file within README.md.

* Update.

* Update.

* 1. Update configuration example file within REAMDME.md for Python version.
2. implement the config DEVICE_PROPERTIES value convertation between the string type and dictionary of Python type.
3. Update the configuration file loading and dumping logic.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Update.

* Update.

* Update.

* Update.

* Update.

* 1. Enable configs to be interchangeable between C++ and Python.
2. Update perf_count showing logic.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Revert the logic of showing show performance counters.

* Update help msg for loading config option.

---------

Signed-off-by: Wang, Yang <yang4.wang@intel.com>
This commit is contained in:
Wang, Yang
2023-04-12 23:32:54 +08:00
committed by GitHub
parent c283d21215
commit 86d7c97fa9
9 changed files with 158 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
tools/benchmark_tool/README.md
View File

@@ -306,26 +306,17 @@ Running the application with the ``-h`` or ``--help`` option yields the followin
Optional. Path to JSON file to load custom OpenVINO parameters.
Please note, command line parameters have higher priority then parameters from configuration file.
Example 1: a simple JSON file for HW device with primary properties.
{
"CPU": {"NUM_STREAMS": "3", "PERF_COUNT": "NO"}
}
{
"CPU": {"NUM_STREAMS": "3", "PERF_COUNT": "NO"}
}
Example 2: a simple JSON file for meta device(AUTO/MULTI) with HW device properties.
{
"AUTO": {
"PERFORMANCE_HINT": "",
"PERF_COUNT": "NO",
"DEVICE_PROPERTIES": {
"CPU": {
"INFERENCE_PRECISION_HINT": "f32",
"NUM_STREAMS": "3"
},
"GPU": {
"INFERENCE_PRECISION_HINT": "f32",
"NUM_STREAMS": "5"
}
}
}
}
{
"AUTO": {
"PERFORMANCE_HINT": "THROUGHPUT",
"PERF_COUNT": "NO",
"DEVICE_PROPERTIES": "{CPU:{INFERENCE_PRECISION_HINT:f32,NUM_STREAMS:3},GPU:{INFERENCE_PRECISION_HINT:f32,NUM_STREAMS:5}}"
}
}
Running the application with the empty list of options yields the usage message given above and an error message.

30
tools/benchmark_tool/openvino/tools/benchmark/main.py
View File

@@ -71,15 +71,12 @@ def main():
device_name = args.target_device
devices = parse_devices(device_name)
is_dev_set_property = {device: True for device in devices}
device_number_streams = parse_value_per_device(devices, args.number_streams, "nstreams")
device_infer_precision = parse_value_per_device(devices, args.infer_precision, "infer_precision")
config = {}
is_load_config = False
if args.load_config:
load_config(args.load_config, config)
is_load_config = True
if is_network_compiled:
logger.info("Model is compiled")
@@ -108,7 +105,7 @@ def main():
# --------------------- 3. Setting device configuration --------------------------------------------------------
next_step()
def get_performance_hint(device) -> properties.hint.PerformanceMode:
def set_performance_hint(device):
perf_hint = properties.hint.PerformanceMode.UNDEFINED
supported_properties = benchmark.core.get_property(device, properties.supported_properties())
if properties.hint.performance_mode() in supported_properties:
@@ -128,9 +125,10 @@ def main():
perf_hint = properties.hint.PerformanceMode.THROUGHPUT if benchmark.api_type == "async" else properties.hint.PerformanceMode.LATENCY
logger.warning(f"Performance hint was not explicitly specified in command line. " +
f"Device({device}) performance hint will be set to {perf_hint}.")
config[device][properties.hint.performance_mode()] = perf_hint
else:
logger.warning(f"Device {device} does not support performance hint property(-hint).")
return perf_hint
def get_device_type_from_name(name) :
new_name = str(name)
@@ -169,10 +167,7 @@ def main():
config[device] = {}
## high-level performance modes
if properties.hint.performance_mode() not in config[device].keys():
config[device][properties.hint.performance_mode()] = get_performance_hint(device)
perf_hint = config[device][properties.hint.performance_mode()]
set_performance_hint(device)
if is_flag_set_in_command_line('nireq'):
config[device][properties.hint.num_requests()] = str(args.number_infer_requests)
@@ -205,14 +200,6 @@ def main():
## insert or append property into hw device properties list
def update_configs(hw_device, property_name, property_value):
(key, value) = properties.device.properties({hw_device:{property_name:property_value}})
is_set_streams_auto = property_name == properties.num_streams() and property_value == properties.streams.Num.AUTO
if not is_set_streams_auto and is_load_config and is_dev_set_property[hw_device] and hw_device in config[device].keys():
# overwrite the device properties loaded from configuration file if
# 1. not setting 'NUM_STREAMS' to default value 'AUTO',
# 2. enable loading device properties from configuration file,
# 3. device properties in config[device] is loaded from configuration file, and never setting device properties before
is_dev_set_property[hw_device] = False
del config[device][key]
# add property into hw device properties list.
if key not in config[device].keys():
config[device][key] = value
@@ -221,10 +208,10 @@ def main():
if hw_device not in current_config.keys():
current_config.update(value.get())
else:
current_device_config = current_config[hw_device].get()
current_device_config = current_config[hw_device]
for prop in value.get().items():
current_device_config.update(prop[1].get())
current_config[hw_device].set(current_device_config)
current_device_config.update(prop[1])
current_config[hw_device].update(current_device_config)
config[device][key].set(current_config)
def update_device_config_for_virtual_device(value, config, key):
@@ -326,7 +313,6 @@ def main():
if device in device_number_streams.keys():
del device_number_streams[device]
perf_counts = perf_counts
device_config = {}
for device in config:
if benchmark.device.find(device) == 0:
@@ -466,7 +452,7 @@ def main():
if k == properties.device.properties():
for device_key in value.keys():
logger.info(f' {device_key}:')
for k2, value2 in value.get(device_key).get().items():
for k2, value2 in value.get(device_key).items():
if k2 not in skip_keys:
logger.info(f' {k2}: {value2}')
else:

13
tools/benchmark_tool/openvino/tools/benchmark/parameters.py
View File

@@ -189,18 +189,9 @@ def parse_args():
"Example 2: a simple JSON file for meta device(AUTO/MULTI) with HW device properties.\n"
" {\n"
" \"AUTO\": {\n"
" \"PERFORMANCE_HINT\": \"\",\n"
" \"PERFORMANCE_HINT\": \"THROUGHPUT\",\n"
" \"PERF_COUNT\": \"NO\",\n"
" \"DEVICE_PROPERTIES\": {\n"
" \"CPU\": {\n"
" \"INFERENCE_PRECISION_HINT\": \"f32\",\n"
" \"NUM_STREAMS\": \"3\"\n"
" },\n"
" \"GPU\": {\n"
" \"INFERENCE_PRECISION_HINT\": \"f32\",\n"
" \"NUM_STREAMS\": \"5\"\n"
" }\n"
" }\n"
" \"DEVICE_PROPERTIES\": \"{CPU:{INFERENCE_PRECISION_HINT:f32,NUM_STREAMS:3},GPU:{INFERENCE_PRECISION_HINT:f32,NUM_STREAMS:5}}\"\n"
" }\n"
" }")
parsed_args = parser.parse_args()

63
tools/benchmark_tool/openvino/tools/benchmark/utils/utils.py
View File

@@ -4,7 +4,7 @@
from collections import defaultdict
from datetime import timedelta
import enum
from openvino.runtime import Core, Model, PartialShape, Dimension, Layout, Type, serialize, properties
from openvino.runtime import Core, Model, PartialShape, Dimension, Layout, Type, serialize, properties, OVAny
from openvino.preprocess import PrePostProcessor
from .constants import DEVICE_DURATION_IN_SECS, UNKNOWN_DEVICE_TYPE, \
@@ -758,12 +758,64 @@ def show_available_devices():
print("\nAvailable target devices: ", (" ".join(Core().available_devices)))
def device_properties_to_string(config):
ret = "{"
for k, v in config.items():
if isinstance(v, dict):
sub_str = "{"
for sk, sv in v.items():
if isinstance(sv, bool):
sv = "YES" if sv else "NO"
if isinstance(sv, properties.Affinity):
sv = sv.name
sub_str += "{0}:{1},".format(sk, sv)
sub_str = sub_str[:-1]
sub_str += "}"
ret += "{0}:{1},".format(k, sub_str)
else:
ret += "{0}:{1},".format(k, v)
ret = ret[:-1]
ret += "}"
return ret
def string_to_device_properties(device_properties_str):
ret = {}
if not device_properties_str:
return ret
if not device_properties_str.startswith("{") or not device_properties_str.endswith("}"):
raise Exception(
"Failed to parse device properties. Value of device properties should be started with '{' and ended with '}'."
"They are actually {} and {}".format(device_properties_str[0], device_properties_str[-1]))
pattern = r'(\w+):({.+?}|[^,}]+)'
pairs = re.findall(pattern, device_properties_str)
for key, value in pairs:
if value.startswith("{") and value.endswith("}"):
value = value[1:-1]
nested_pairs = re.findall(pattern, value)
nested_dict = {}
for nested_key, nested_value in nested_pairs:
nested_dict[nested_key] = nested_value
value = nested_dict
ret[key] = value
return ret
def dump_config(filename, config):
json_config = {}
for device_name, device_config in config.items():
json_config[device_name] = {}
for key, value in device_config.items():
value_string = value.name if isinstance(value, properties.hint.PerformanceMode) else str(value)
if isinstance(value, OVAny) and (isinstance(value.value, dict)):
value_string = device_properties_to_string(value.get())
elif isinstance(value, (properties.hint.PerformanceMode, properties.Affinity)):
value_string = value.name
elif isinstance(value, OVAny):
value_string = str(value.value)
else:
value_string = str(value)
if isinstance(value, bool):
value_string = "YES" if value else "NO"
json_config[device_name][key] = value_string
with open(filename, 'w') as f:
@@ -776,4 +828,9 @@ def load_config(filename, config):
for device in original_config:
config[device] = {}
for property_name in original_config[device]:
config[device][property_name] = original_config[device][property_name]
property_value = original_config[device][property_name]
if property_name == properties.device.properties():
property_value = string_to_device_properties(property_value)
elif property_value in ("YES", "NO"):
property_value = True if property_value == "YES" else False
config[device][property_name] = OVAny(property_value)