diff --git a/tools/benchmark_tool/openvino/tools/benchmark/main.py b/tools/benchmark_tool/openvino/tools/benchmark/main.py index 47fabb34c48..422dc0d3991 100644 --- a/tools/benchmark_tool/openvino/tools/benchmark/main.py +++ b/tools/benchmark_tool/openvino/tools/benchmark/main.py @@ -334,7 +334,10 @@ def run(args): paths_to_input = list() if args.paths_to_input: for path in args.paths_to_input: - paths_to_input.append(os.path.abspath(*path) if args.paths_to_input else None) + if ":" in next(iter(path), ""): + paths_to_input.extend(path) + else: + paths_to_input.append(os.path.abspath(*path)) set_inputs(paths_to_input, batch_size, app_inputs_info, infer_requests) if statistics: diff --git a/tools/benchmark_tool/openvino/tools/benchmark/parameters.py b/tools/benchmark_tool/openvino/tools/benchmark/parameters.py index 89a3c6283dc..f7f42b1c16c 100644 --- a/tools/benchmark_tool/openvino/tools/benchmark/parameters.py +++ b/tools/benchmark_tool/openvino/tools/benchmark/parameters.py @@ -33,7 +33,9 @@ def parse_args(): help='Show this help message and exit.') args.add_argument('-i', '--paths_to_input', action='append', nargs='+', type=str, required=False, help='Optional. ' - 'Path to a folder with images and/or binaries or to specific image or binary file.') + 'Path to a folder with images and/or binaries or to specific image or binary file.' + 'It is also allowed to map files to network inputs: ' + 'input_1:file_1,file_2,input_4:file_4 input_2:file_3') args.add_argument('-m', '--path_to_model', type=str, required=True, help='Required. Path to an .xml/.onnx file with a trained model or ' 'to a .blob file with a trained compiled model.') diff --git a/tools/benchmark_tool/openvino/tools/benchmark/utils/inputs_filling.py b/tools/benchmark_tool/openvino/tools/benchmark/utils/inputs_filling.py index 0566b9b6ea8..dcae3f0aefa 100644 --- a/tools/benchmark_tool/openvino/tools/benchmark/utils/inputs_filling.py +++ b/tools/benchmark_tool/openvino/tools/benchmark/utils/inputs_filling.py @@ -3,12 +3,17 @@ import os import cv2 +import re import numpy as np from glob import glob +from collections import defaultdict +from pathlib import Path +from itertools import chain from .constants import IMAGE_EXTENSIONS, BINARY_EXTENSIONS from .logging import logger + def set_inputs(paths_to_input, batch_size, app_input_info, requests): requests_input_data = get_inputs(paths_to_input, batch_size, app_input_info, requests) for i in range(len(requests)): @@ -18,7 +23,11 @@ def set_inputs(paths_to_input, batch_size, app_input_info, requests): raise Exception(f"No input with name {k} found!") inputs[k].buffer[:] = v + def get_inputs(paths_to_input, batch_size, app_input_info, requests): + input_file_mapping = parse_paths_to_input(paths_to_input) + check_input_file_mapping(input_file_mapping, app_input_info) + input_image_sizes = {} for key in sorted(app_input_info.keys()): info = app_input_info[key] @@ -31,13 +40,15 @@ def get_inputs(paths_to_input, batch_size, app_input_info, requests): image_files = list() binary_files = list() - if paths_to_input: + if paths_to_input and not input_file_mapping: image_files = get_files_by_extensions(paths_to_input, IMAGE_EXTENSIONS) - image_files.sort() binary_files = get_files_by_extensions(paths_to_input, BINARY_EXTENSIONS) - binary_files.sort() - if (len(image_files) == 0) and (len(binary_files) == 0): + if input_file_mapping and len(input_file_mapping) < len(app_input_info): + not_provided_inputs = set(app_input_info) - set(input_file_mapping) + logger.warning("No input files were given for the inputs: " + f"{', '.join(not_provided_inputs)}. This inputs will be filled with random values!") + elif (len(image_files) == 0) and (len(binary_files) == 0): logger.warning("No input files were given: all inputs will be filled with random values!") else: binary_to_be_used = binaries_count * batch_size * len(requests) @@ -76,13 +87,23 @@ def get_inputs(paths_to_input, batch_size, app_input_info, requests): info = app_input_info[key] if info.is_image: # input is image + if key in input_file_mapping: + input_data[key] = fill_blob_with_image(input_file_mapping[key], request_id, batch_size, + keys.index(key), len(keys), info, from_map=True) + continue + if len(image_files) > 0: input_data[key] = fill_blob_with_image(image_files, request_id, batch_size, keys.index(key), len(keys), info) continue # input is binary - if len(binary_files): + if len(binary_files) or key in input_file_mapping: + if key in input_file_mapping: + input_data[key] = fill_blob_with_binary(input_file_mapping[key], request_id, batch_size, + keys.index(key), len(keys), info, from_map=True) + continue + input_data[key] = fill_blob_with_binary(binary_files, request_id, batch_size, keys.index(key), len(keys), info) continue @@ -106,9 +127,19 @@ def get_inputs(paths_to_input, batch_size, app_input_info, requests): def get_files_by_extensions(paths_to_input, extensions): - get_extension = lambda file_path: file_path.split(".")[-1].upper() + if len(paths_to_input) == 1: + files = [file for file in paths_to_input[0].split(",") if file] + if all(get_extension(file) in extensions for file in files): + check_files_exist(files) + return files + + return get_files_by_extensions_for_not_list_of_files(paths_to_input, extensions) + + +def get_files_by_extensions_for_not_list_of_files(paths_to_input, extensions): input_files = list() + for path_to_input in paths_to_input: if os.path.isfile(path_to_input): files = [os.path.normpath(path_to_input)] @@ -119,13 +150,21 @@ def get_files_by_extensions(paths_to_input, extensions): file_extension = get_extension(file) if file_extension in extensions: input_files.append(file) + input_files.sort() + return input_files - return input_files -def fill_blob_with_image(image_paths, request_id, batch_size, input_id, input_size, info): +def get_extension(file_path): + return file_path.split(".")[-1].upper() + + +def fill_blob_with_image(image_paths, request_id, batch_size, input_id, input_size, info, from_map=False): shape = info.shape images = np.ndarray(shape) - image_index = request_id * batch_size * input_size + input_id + if from_map: + image_index = request_id * batch_size + else: + image_index = request_id * batch_size * input_size + input_id scale_mean = (not np.array_equal(info.scale, (1.0, 1.0, 1.0)) or not np.array_equal(info.mean, (0.0, 0.0, 0.0))) @@ -154,7 +193,10 @@ def fill_blob_with_image(image_paths, request_id, batch_size, input_id, input_si images[b] = image - image_index += input_size + if from_map: + image_index += 1 + else: + image_index += input_size return images def get_dtype(precision): @@ -173,12 +215,15 @@ def get_dtype(precision): return format_map[precision] raise Exception("Can't find data type for precision: " + precision) -def fill_blob_with_binary(binary_paths, request_id, batch_size, input_id, input_size, info): +def fill_blob_with_binary(binary_paths, request_id, batch_size, input_id, input_size, info, from_map=False): binaries = np.ndarray(info.shape) shape = info.shape.copy() if 'N' in info.layout: shape[info.layout.index('N')] = 1 - binary_index = request_id * batch_size * input_size + input_id + if from_map: + binary_index = request_id * batch_size + else: + binary_index = request_id * batch_size * input_size + input_id dtype = get_dtype(info.precision)[0] for b in range(batch_size): binary_index %= len(binary_paths) @@ -191,7 +236,11 @@ def fill_blob_with_binary(binary_paths, request_id, batch_size, input_id, input_ raise Exception( f"File {binary_filename} contains {binary_file_size} bytes but network expects {blob_size}") binaries[b] = np.reshape(np.fromfile(binary_filename, dtype), shape) - binary_index += input_size + + if from_map: + binary_index += 1 + else: + binary_index += input_size return binaries @@ -214,3 +263,75 @@ def fill_blob_with_random(layer): if layer.shape: return rs.uniform(rand_min, rand_max, layer.shape).astype(dtype) return (dtype)(rs.uniform(rand_min, rand_max)) + + +def parse_paths_to_input(paths_to_inputs): + input_dicts_list = [parse_path(path) for path in paths_to_inputs] + inputs = defaultdict(list) + for input_dict in input_dicts_list: + for input_name, input_files in input_dict.items(): + inputs[input_name] += input_files + return {input_: files for input_, files in inputs.items() if files} + + +def parse_path(path): + """ + Parse "input_1:file1,file2,input_2:file3" into a dict + """ + inputs = re.findall(r"([^,]\w+):", path) + input_files = [file for file in re.split(r"[^,]\w+:", path) if file] + return { + input_: files.strip(",").split(",") for input_, files in zip(inputs, input_files) + } + + +def check_input_file_mapping(input_file_mapping, app_input_info): + check_inputs(app_input_info, input_file_mapping) + check_input_file_mapping_files_exists(input_file_mapping) + check_files_extensions(app_input_info, input_file_mapping) + + +def check_inputs(app_input_info, input_file_mapping): + wrong_inputs = [ + input_ for input_ in input_file_mapping if input_ not in app_input_info + ] + if wrong_inputs: + raise Exception( + f"Wrong input mapping! Cannot find inputs: {wrong_inputs}. " + f"Available inputs: {list(app_input_info)}. " + "Please check `-i` input data" + ) + + +def check_input_file_mapping_files_exists(input_file_mapping): + check_files_exist(chain.from_iterable(input_file_mapping.values())) + + +def check_files_exist(input_files_list): + not_files = [ + file for file in input_files_list if not Path(file).is_file() + ] + if not_files: + not_files = ",\n".join(not_files) + raise Exception( + f"Inputs are not files or does not exist!\n {not_files}" + ) + + +def check_files_extensions(app_input_info, input_file_mapping): + unsupported_files = [] + for input_, files in input_file_mapping.items(): + info = app_input_info[input_] + + proper_extentions = IMAGE_EXTENSIONS if info.is_image else BINARY_EXTENSIONS + unsupported = "\n".join( + [file for file in files if Path(file).suffix.upper().strip(".") not in proper_extentions] + ) + if unsupported: + unsupported_files.append(unsupported) + if unsupported_files: + unsupported_files = "\n".join(unsupported_files) + raise Exception( + f"This files has unsupported extensions: {unsupported_files}.\n" + f"Supported extentions:\nImages: {IMAGE_EXTENSIONS}\nBinary: {BINARY_EXTENSIONS}" + )