IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add input to file mappings to python benchmark tool (#8210)

Browse Source 
* Add inputs-to-files path to benchmark_tool

* Add inputs-to-files path to benchmark_tool

* Add inputs-to-files path to benchmark_tool

* Remove `<>`

* Fix binary input fill

* Remove redundant checks

Change `:` delimiter to `::`

* Delete print

* Change delimiter to `:`
Add support for list of files

* Add warning if some inputs filled with random values
This commit is contained in:
Artur Paniukov
2021-11-09 17:56:37 +04:00
committed by GitHub
parent 6c3e68a8d1
commit a9b9e14bf4
3 changed files with 141 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tools/benchmark_tool/openvino/tools/benchmark/main.py
View File

@@ -334,7 +334,10 @@ def run(args):
paths_to_input = list()
if args.paths_to_input:
for path in args.paths_to_input:
paths_to_input.append(os.path.abspath(*path) if args.paths_to_input else None)
if ":" in next(iter(path), ""):
paths_to_input.extend(path)
else:
paths_to_input.append(os.path.abspath(*path))
set_inputs(paths_to_input, batch_size, app_inputs_info, infer_requests)
if statistics:

4
tools/benchmark_tool/openvino/tools/benchmark/parameters.py
View File

@@ -33,7 +33,9 @@ def parse_args():
help='Show this help message and exit.')
args.add_argument('-i', '--paths_to_input', action='append', nargs='+', type=str, required=False,
help='Optional. '
'Path to a folder with images and/or binaries or to specific image or binary file.')
'Path to a folder with images and/or binaries or to specific image or binary file.'
'It is also allowed to map files to network inputs: '
'input_1:file_1,file_2,input_4:file_4 input_2:file_3')
args.add_argument('-m', '--path_to_model', type=str, required=True,
help='Required. Path to an .xml/.onnx file with a trained model or '
'to a .blob file with a trained compiled model.')

147
tools/benchmark_tool/openvino/tools/benchmark/utils/inputs_filling.py
View File

@@ -3,12 +3,17 @@
import os
import cv2
import re
import numpy as np
from glob import glob
from collections import defaultdict
from pathlib import Path
from itertools import chain
from .constants import IMAGE_EXTENSIONS, BINARY_EXTENSIONS
from .logging import logger
def set_inputs(paths_to_input, batch_size, app_input_info, requests):
requests_input_data = get_inputs(paths_to_input, batch_size, app_input_info, requests)
for i in range(len(requests)):
@@ -18,7 +23,11 @@ def set_inputs(paths_to_input, batch_size, app_input_info, requests):
raise Exception(f"No input with name {k} found!")
inputs[k].buffer[:] = v
def get_inputs(paths_to_input, batch_size, app_input_info, requests):
input_file_mapping = parse_paths_to_input(paths_to_input)
check_input_file_mapping(input_file_mapping, app_input_info)
input_image_sizes = {}
for key in sorted(app_input_info.keys()):
info = app_input_info[key]
@@ -31,13 +40,15 @@ def get_inputs(paths_to_input, batch_size, app_input_info, requests):
image_files = list()
binary_files = list()
if paths_to_input:
if paths_to_input and not input_file_mapping:
image_files = get_files_by_extensions(paths_to_input, IMAGE_EXTENSIONS)
image_files.sort()
binary_files = get_files_by_extensions(paths_to_input, BINARY_EXTENSIONS)
binary_files.sort()
if (len(image_files) == 0) and (len(binary_files) == 0):
if input_file_mapping and len(input_file_mapping) < len(app_input_info):
not_provided_inputs = set(app_input_info) - set(input_file_mapping)
logger.warning("No input files were given for the inputs: "
f"{', '.join(not_provided_inputs)}. This inputs will be filled with random values!")
elif (len(image_files) == 0) and (len(binary_files) == 0):
logger.warning("No input files were given: all inputs will be filled with random values!")
else:
binary_to_be_used = binaries_count * batch_size * len(requests)
@@ -76,13 +87,23 @@ def get_inputs(paths_to_input, batch_size, app_input_info, requests):
info = app_input_info[key]
if info.is_image:
# input is image
if key in input_file_mapping:
input_data[key] = fill_blob_with_image(input_file_mapping[key], request_id, batch_size,
keys.index(key), len(keys), info, from_map=True)
continue
if len(image_files) > 0:
input_data[key] = fill_blob_with_image(image_files, request_id, batch_size, keys.index(key),
len(keys), info)
continue
# input is binary
if len(binary_files):
if len(binary_files) or key in input_file_mapping:
if key in input_file_mapping:
input_data[key] = fill_blob_with_binary(input_file_mapping[key], request_id, batch_size,
keys.index(key), len(keys), info, from_map=True)
continue
input_data[key] = fill_blob_with_binary(binary_files, request_id, batch_size, keys.index(key),
len(keys), info)
continue
@@ -106,9 +127,19 @@ def get_inputs(paths_to_input, batch_size, app_input_info, requests):
def get_files_by_extensions(paths_to_input, extensions):
get_extension = lambda file_path: file_path.split(".")[-1].upper()
if len(paths_to_input) == 1:
files = [file for file in paths_to_input[0].split(",") if file]
if all(get_extension(file) in extensions for file in files):
check_files_exist(files)
return files
return get_files_by_extensions_for_not_list_of_files(paths_to_input, extensions)
def get_files_by_extensions_for_not_list_of_files(paths_to_input, extensions):
input_files = list()
for path_to_input in paths_to_input:
if os.path.isfile(path_to_input):
files = [os.path.normpath(path_to_input)]
@@ -119,13 +150,21 @@ def get_files_by_extensions(paths_to_input, extensions):
file_extension = get_extension(file)
if file_extension in extensions:
input_files.append(file)
input_files.sort()
return input_files
return input_files
def fill_blob_with_image(image_paths, request_id, batch_size, input_id, input_size, info):
def get_extension(file_path):
return file_path.split(".")[-1].upper()
def fill_blob_with_image(image_paths, request_id, batch_size, input_id, input_size, info, from_map=False):
shape = info.shape
images = np.ndarray(shape)
image_index = request_id * batch_size * input_size + input_id
if from_map:
image_index = request_id * batch_size
else:
image_index = request_id * batch_size * input_size + input_id
scale_mean = (not np.array_equal(info.scale, (1.0, 1.0, 1.0)) or not np.array_equal(info.mean, (0.0, 0.0, 0.0)))
@@ -154,7 +193,10 @@ def fill_blob_with_image(image_paths, request_id, batch_size, input_id, input_si
images[b] = image
image_index += input_size
if from_map:
image_index += 1
else:
image_index += input_size
return images
def get_dtype(precision):
@@ -173,12 +215,15 @@ def get_dtype(precision):
return format_map[precision]
raise Exception("Can't find data type for precision: " + precision)
def fill_blob_with_binary(binary_paths, request_id, batch_size, input_id, input_size, info):
def fill_blob_with_binary(binary_paths, request_id, batch_size, input_id, input_size, info, from_map=False):
binaries = np.ndarray(info.shape)
shape = info.shape.copy()
if 'N' in info.layout:
shape[info.layout.index('N')] = 1
binary_index = request_id * batch_size * input_size + input_id
if from_map:
binary_index = request_id * batch_size
else:
binary_index = request_id * batch_size * input_size + input_id
dtype = get_dtype(info.precision)[0]
for b in range(batch_size):
binary_index %= len(binary_paths)
@@ -191,7 +236,11 @@ def fill_blob_with_binary(binary_paths, request_id, batch_size, input_id, input_
raise Exception(
f"File {binary_filename} contains {binary_file_size} bytes but network expects {blob_size}")
binaries[b] = np.reshape(np.fromfile(binary_filename, dtype), shape)
binary_index += input_size
if from_map:
binary_index += 1
else:
binary_index += input_size
return binaries
@@ -214,3 +263,75 @@ def fill_blob_with_random(layer):
if layer.shape:
return rs.uniform(rand_min, rand_max, layer.shape).astype(dtype)
return (dtype)(rs.uniform(rand_min, rand_max))
def parse_paths_to_input(paths_to_inputs):
input_dicts_list = [parse_path(path) for path in paths_to_inputs]
inputs = defaultdict(list)
for input_dict in input_dicts_list:
for input_name, input_files in input_dict.items():
inputs[input_name] += input_files
return {input_: files for input_, files in inputs.items() if files}
def parse_path(path):
"""
Parse "input_1:file1,file2,input_2:file3" into a dict
"""
inputs = re.findall(r"([^,]\w+):", path)
input_files = [file for file in re.split(r"[^,]\w+:", path) if file]
return {
input_: files.strip(",").split(",") for input_, files in zip(inputs, input_files)
}
def check_input_file_mapping(input_file_mapping, app_input_info):
check_inputs(app_input_info, input_file_mapping)
check_input_file_mapping_files_exists(input_file_mapping)
check_files_extensions(app_input_info, input_file_mapping)
def check_inputs(app_input_info, input_file_mapping):
wrong_inputs = [
input_ for input_ in input_file_mapping if input_ not in app_input_info
]
if wrong_inputs:
raise Exception(
f"Wrong input mapping! Cannot find inputs: {wrong_inputs}. "
f"Available inputs: {list(app_input_info)}. "
"Please check `-i` input data"
)
def check_input_file_mapping_files_exists(input_file_mapping):
check_files_exist(chain.from_iterable(input_file_mapping.values()))
def check_files_exist(input_files_list):
not_files = [
file for file in input_files_list if not Path(file).is_file()
]
if not_files:
not_files = ",\n".join(not_files)
raise Exception(
f"Inputs are not files or does not exist!\n {not_files}"
)
def check_files_extensions(app_input_info, input_file_mapping):
unsupported_files = []
for input_, files in input_file_mapping.items():
info = app_input_info[input_]
proper_extentions = IMAGE_EXTENSIONS if info.is_image else BINARY_EXTENSIONS
unsupported = "\n".join(
[file for file in files if Path(file).suffix.upper().strip(".") not in proper_extentions]
)
if unsupported:
unsupported_files.append(unsupported)
if unsupported_files:
unsupported_files = "\n".join(unsupported_files)
raise Exception(
f"This files has unsupported extensions: {unsupported_files}.\n"
f"Supported extentions:\nImages: {IMAGE_EXTENSIONS}\nBinary: {BINARY_EXTENSIONS}"
)