Adds support of user layouts to benchmark_app (#4002)

* Adds support of user layouts to benchmark_app

* Keep snake_case for python

Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com>

tools/benchmark/README.md

@@ -103,15 +103,19 @@ Options:
   -progress [PROGRESS]  Optional. Show progress bar (can affect performance
                         measurement). Default values is "False".
   -shape SHAPE          Optional. Set shape for input. For example,
-                        "input1[1,3,224,224],input2[1,4]" or "[1,3,224,224]" in
-                        case of one input size.
+                        "input1[1,3,224,224],input2[1,4]" or "[1,3,224,224]"
+                        in case of one input size.
+  -layout LAYOUT        Optional. Prompts how network layouts should be
+                        treated by application. For example,
+                        "input1[NCHW],input2[NC]" or "[NCHW]" in case of one
+                        input size.
   -nstreams NUMBER_STREAMS, --number_streams NUMBER_STREAMS
                        Optional. Number of streams to use for inference on the CPU/GPU/MYRIAD
                        (for HETERO and MULTI device cases use format <device1>:<nstreams1>,<device2>:<nstreams2> or just <nstreams>).
-                       Default value is determined automatically for a device. 
-                       Please note that although the automatic selection usually provides a reasonable performance, 
+                       Default value is determined automatically for a device.
+                       Please note that although the automatic selection usually provides a reasonable performance,
                        it still may be non-optimal for some cases, especially for very small networks.
-                       Also, using nstreams>1 is inherently throughput-oriented option, while for the best-latency 
+                       Also, using nstreams>1 is inherently throughput-oriented option, while for the best-latency
                        estimations the number of streams should be set to 1.
 -enforcebf16 [ENFORCE_BFLOAT16], --enforce_bfloat16 [ENFORCE_BFLOAT16]
                         Optional. Enforcing of floating point operations

tools/benchmark/main.py

@@ -11,8 +11,8 @@ from openvino.tools.benchmark.utils.logging import logger
 from openvino.tools.benchmark.utils.progress_bar import ProgressBar
 from openvino.tools.benchmark.utils.utils import next_step, config_network_inputs, get_number_iterations, \
     process_help_inference_string, print_perf_counters, dump_exec_graph, get_duration_in_milliseconds, \
-    get_command_line_arguments, parse_nstreams_value_per_device, parse_devices, update_shapes, \
-    adjust_shapes_batch, load_config, dump_config
+    get_command_line_arguments, parse_nstreams_value_per_device, parse_devices, get_inputs_info, \
+    get_batch_size, load_config, dump_config
 from openvino.tools.benchmark.utils.statistics_report import StatisticsReport, averageCntReport, detailedCntReport
 
 
@@ -193,15 +193,10 @@ def run(args):
             # --------------------- 5. Resizing network to match image sizes and given batch ---------------------------
             next_step()
 
-            shapes = {k: v.input_data.shape.copy() for k, v in ie_network.input_info.items()}
-            reshape = False
-            if args.shape:
-                reshape |= update_shapes(shapes, args.shape, ie_network.input_info)
-            if args.batch_size and args.batch_size != ie_network.batch_size:
-                reshape |= adjust_shapes_batch(shapes, args.batch_size, ie_network.input_info)
-
+            app_inputs_info, reshape = get_inputs_info(args.shape, args.layout, args.batch_size, ie_network.input_info)
             if reshape:
                 start_time = datetime.utcnow()
+                shapes = { k : v.shape for k,v in app_inputs_info.items() }
                 logger.info(
                     'Reshaping network: {}'.format(', '.join("'{}': {}".format(k, v) for k, v in shapes.items())))
                 ie_network.reshape(shapes)
@@ -213,13 +208,15 @@ def run(args):
                                                   ('reshape network time (ms)', duration_ms)
                                               ])
 
-            batch_size = ie_network.batch_size
-            logger.info('Network batch size: {}'.format(ie_network.batch_size))
+            # use batch size according to provided layout and shapes
+            batch_size = get_batch_size(app_inputs_info) if args.layout else ie_network.batch_size
+
+            logger.info('Network batch size: {}'.format(batch_size))
 
             # --------------------- 6. Configuring input of the model --------------------------------------------------
             next_step()
 
-            config_network_inputs(ie_network)
+            config_network_inputs(ie_network, app_inputs_info)
 
             # --------------------- 7. Loading the model to the device -------------------------------------------------
             next_step()
@@ -253,6 +250,7 @@ def run(args):
                                           [
                                               ('import network time (ms)', duration_ms)
                                           ])
+            app_inputs_info, _ = get_inputs_info(args.shape, args.layout, args.batch_size, exe_network.input_info)
             if batch_size == 0:
                 batch_size = 1
 
@@ -277,7 +275,7 @@ def run(args):
         if args.paths_to_input:
             for path in args.paths_to_input:
                 paths_to_input.append(os.path.abspath(*path) if args.paths_to_input else None)
-        set_inputs(paths_to_input, batch_size, exe_network.input_info, infer_requests)
+        set_inputs(paths_to_input, batch_size, app_inputs_info, infer_requests)
 
         if statistics:
             statistics.add_parameters(StatisticsReport.Category.RUNTIME_CONFIG,

tools/benchmark/parameters.py

@@ -68,6 +68,10 @@ def parse_args():
     args.add_argument('-shape', type=str, required=False, default='',
                       help='Optional. '
                            'Set shape for input. For example, "input1[1,3,224,224],input2[1,4]" or "[1,3,224,224]" in case of one input size.')
+    args.add_argument('-layout', type=str, required=False, default='',
+                      help='Optional. '
+                           'Prompts how network layouts should be treated by application. '
+                           'For example, "input1[NCHW],input2[NC]" or "[NCHW]" in case of one input size.')
     args.add_argument('-nstreams', '--number_streams', type=str, required=False, default=None,
                       help='Optional. Number of streams to use for inference on the CPU/GPU/MYRIAD '
                            '(for HETERO and MULTI device cases use format <device1>:<nstreams1>,<device2>:<nstreams2> '
@@ -83,7 +87,7 @@ def parse_args():
                       help='Number of threads to use for inference on the CPU, GNA '
                            '(including HETERO and MULTI cases).')
     args.add_argument('-pin', '--infer_threads_pinning', type=str, required=False, default='YES', choices=['YES', 'NO', 'NUMA'],
-                      help='Optional. Enable  threads->cores (\'YES\' is default value), threads->(NUMA)nodes (\'NUMA\') or completely  disable (\'NO\')' 
+                      help='Optional. Enable  threads->cores (\'YES\' is default value), threads->(NUMA)nodes (\'NUMA\') or completely  disable (\'NO\')'
                            'CPU threads pinning for CPU-involved inference.')
     args.add_argument('-exec_graph_path', '--exec_graph_path', type=str, required=False,
                       help='Optional. Path to a file where to store executable graph information serialized.')

tools/benchmark/utils/inputs_filling.py

@@ -1,5 +1,5 @@
 """
- Copyright (C) 2018-2020 Intel Corporation
+ Copyright (C) 2018-2021 Intel Corporation
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
@@ -22,21 +22,8 @@ from glob import glob
 from .constants import IMAGE_EXTENSIONS, BINARY_EXTENSIONS
 from .logging import logger
 
-def is_image(blob):
-    if blob.layout != "NCHW":
-        return False
-    channels = blob.shape[1]
-    return channels == 3
-
-
-def is_image_info(blob):
-    if blob.layout != "NC":
-        return False
-    channels = blob.shape[1]
-    return channels >= 2
-
-def set_inputs(paths_to_input, batch_size, input_info, requests):
-  requests_input_data = get_inputs(paths_to_input, batch_size, input_info, requests)
+def set_inputs(paths_to_input, batch_size, app_input_info, requests):
+  requests_input_data = get_inputs(paths_to_input, batch_size, app_input_info, requests)
   for i in range(len(requests)):
     inputs = requests[i].input_blobs
     for k, v in requests_input_data[i].items():
@@ -44,19 +31,20 @@ def set_inputs(paths_to_input, batch_size, input_info, requests):
             raise Exception("No input with name {} found!".format(k))
         inputs[k].buffer[:] = v
 
-def get_inputs(paths_to_input, batch_size, input_info, requests):
+def get_inputs(paths_to_input, batch_size, app_input_info, requests):
     input_image_sizes = {}
-    for key in sorted(input_info.keys()):
-        if is_image(input_info[key].input_data):
-            input_image_sizes[key] = (input_info[key].input_data.shape[2], input_info[key].input_data.shape[3])
+    for key in sorted(app_input_info.keys()):
+        info = app_input_info[key]
+        if info.is_image:
+            input_image_sizes[key] = (info.width, info.height)
         logger.info("Network input '{}' precision {}, dimensions ({}): {}".format(key,
-                                                                                  input_info[key].input_data.precision,
-                                                                                  input_info[key].input_data.layout,
+                                                                                  info.precision,
+                                                                                  info.layout,
                                                                                   " ".join(str(x) for x in
-                                                                                           input_info[key].input_data.shape)))
+                                                                                           info.shape)))
 
     images_count = len(input_image_sizes.keys())
-    binaries_count = len(input_info) - images_count
+    binaries_count = len(app_input_info) - images_count
 
     image_files = list()
     binary_files = list()
@@ -100,33 +88,34 @@ def get_inputs(paths_to_input, batch_size, input_info, requests):
     for request_id in range(0, len(requests)):
         logger.info("Infer Request {} filling".format(request_id))
         input_data = {}
-        keys = list(sorted(input_info.keys()))
+        keys = list(sorted(app_input_info.keys()))
         for key in keys:
-            if is_image(input_info[key].input_data):
+            info = app_input_info[key]
+            if info.is_image:
                 # input is image
                 if len(image_files) > 0:
                     input_data[key] = fill_blob_with_image(image_files, request_id, batch_size, keys.index(key),
-                                                           len(keys), input_info[key].input_data)
+                                                           len(keys), info)
                     continue
 
             # input is binary
             if len(binary_files):
                 input_data[key] = fill_blob_with_binary(binary_files, request_id, batch_size, keys.index(key),
-                                                        len(keys), input_info[key].input_data)
+                                                        len(keys), info)
                 continue
 
             # most likely input is image info
-            if is_image_info(input_info[key].input_data) and len(input_image_sizes) == 1:
+            if info.is_image_info and len(input_image_sizes) == 1:
                 image_size = input_image_sizes[list(input_image_sizes.keys()).pop()]
                 logger.info("Fill input '" + key + "' with image size " + str(image_size[0]) + "x" +
                             str(image_size[1]))
-                input_data[key] = fill_blob_with_image_info(image_size, input_info[key].input_data)
+                input_data[key] = fill_blob_with_image_info(image_size, info)
                 continue
 
             # fill with random data
-            logger.info("Fill input '{}' with random values ({} is expected)".format(key, "image" if is_image(
-                input_info[key].input_data) else "some binary data"))
-            input_data[key] = fill_blob_with_random(input_info[key].input_data)
+            logger.info("Fill input '{}' with random values ({} is expected)".format(key, "image"
+                if info.is_image else "some binary data"))
+            input_data[key] = fill_blob_with_random(info)
 
         requests_input_data.append(input_data)
 
@@ -150,8 +139,8 @@ def get_files_by_extensions(paths_to_input, extensions):
 
     return input_files
 
-def fill_blob_with_image(image_paths, request_id, batch_size, input_id, input_size, layer):
-    shape = layer.shape
+def fill_blob_with_image(image_paths, request_id, batch_size, input_id, input_size, info):
+    shape = info.shape
     images = np.ndarray(shape)
     image_index = request_id * batch_size * input_size + input_id
     for b in range(batch_size):
@@ -159,15 +148,11 @@ def fill_blob_with_image(image_paths, request_id, batch_size, input_id, input_si
         image_filename = image_paths[image_index]
         logger.info('Prepare image {}'.format(image_filename))
         image = cv2.imread(image_filename)
-
-        new_im_size = tuple(shape[2:])
+        new_im_size = tuple((info.width, info.height))
         if image.shape[:-1] != new_im_size:
             logger.warning("Image is resized from ({}) to ({})".format(image.shape[:-1], new_im_size))
             image = cv2.resize(image, new_im_size)
-
-        if image.shape[0] != shape[2]:
-            image = image.transpose((2, 1, 0))
-        else:
+        if info.layout in ['NCHW', 'CHW']:
             image = image.transpose((2, 0, 1))
         images[b] = image
 
@@ -189,11 +174,13 @@ def get_dtype(precision):
         return format_map[precision]
     raise Exception("Can't find data type for precision: " + precision)
 
-def fill_blob_with_binary(binary_paths, request_id, batch_size, input_id, input_size, layer):
-    binaries = np.ndarray(layer.shape)
-    shape = get_blob_shape(layer, 1) # get blob shape for batch 1
+def fill_blob_with_binary(binary_paths, request_id, batch_size, input_id, input_size, info):
+    binaries = np.ndarray(info.shape)
+    shape = info.shape.copy()
+    if 'N' in info.layout:
+        shape[info.layout.index('N')] = 1
     binary_index = request_id * batch_size * input_size + input_id
-    dtype = get_dtype(layer.precision)
+    dtype = get_dtype(info.precision)
     for b in range(batch_size):
         binary_index %= len(binary_paths)
         binary_filename = binary_paths[binary_index]

tools/benchmark/utils/utils.py

@@ -1,5 +1,5 @@
 """
- Copyright (C) 2018-2020 Intel Corporation
+ Copyright (C) 2018-2021 Intel Corporation
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
@@ -17,7 +17,6 @@ from openvino.inference_engine import IENetwork,IECore
 
 from .constants import DEVICE_DURATION_IN_SECS, UNKNOWN_DEVICE_TYPE, \
     CPU_DEVICE_NAME, GPU_DEVICE_NAME
-from .inputs_filling import is_image
 from .logging import logger
 
 import json
@@ -61,13 +60,13 @@ def next_step(additional_info='', step_id=0):
     print(step_info_template)
 
 
-def config_network_inputs(ie_network: IENetwork):
+def config_network_inputs(ie_network: IENetwork, app_inputs_info):
     input_info = ie_network.input_info
-
     for key in input_info.keys():
-        if is_image(input_info[key].input_data):
+        if app_inputs_info[key].is_image:
             # Set the precision of input data provided by the user
             # Should be called before load of the network to the plugin
+            app_inputs_info[key].precision = 'U8'
             input_info[key].precision = 'U8'
 
 
@@ -227,33 +226,105 @@ def get_command_line_arguments(argv):
         parameters.append((arg_name, arg_value))
     return parameters
 
-def update_shapes(shapes, shapes_string: str, inputs_info):
-    updated = False
-    matches = re.findall(r'(.*?)\[(.*?)\],?', shapes_string)
-    if matches:
-        for match in matches:
-            input_name = match[0]
-            parsed_shape = [int(dim) for dim in match[1].split(',')]
-            if input_name != '':
-                shapes[input_name] = parsed_shape
-                updated = True
-            else:
-                shapes.update({ k:parsed_shape for k in shapes.keys() })
-                updated = True
-                break
-    else:
-        raise Exception("Can't parse `shape` parameter: {}".format(shapes_string))
-    return updated
+def parse_input_parameters(parameter_string, input_info):
+    # Parse parameter string like "input0[value0],input1[value1]" or "[value]" (applied to all inputs)
+    return_value = {}
+    if parameter_string:
+        matches = re.findall(r'(.*?)\[(.*?)\],?', parameter_string)
+        if matches:
+            for match in matches:
+                input_name, value = match
+                if input_name != '':
+                    return_value[input_name] = value
+                else:
+                    return_value  = { k:value for k in input_info.keys() }
+                    break
+        else:
+            raise Exception("Can't parse input parameter: {}".format(parameter_string))
+    return return_value
 
-def adjust_shapes_batch(shapes, batch_size: int, inputs_info):
-    updated = False
-    for name, data in inputs_info.items():
-        layout = data.input_data.layout
-        batch_index = layout.index('N') if 'N' in layout else -1
-        if batch_index != -1 and shapes[name][batch_index] != batch_size:
-            shapes[name][batch_index] = batch_size
-            updated = True
-    return updated
+class InputInfo:
+    def __init__(self):
+        self.precision = None
+        self.layout = ""
+        self.shape = []
+
+    @property
+    def is_image(self):
+        if self.layout not in [ "NCHW", "NHWC", "CHW", "HWC" ]:
+            return False
+        return self.channels == 3
+
+    @property
+    def is_image_info(self):
+        if self.layout != "NC":
+            return False
+        return self.channels >= 2
+
+    def getDimentionByLayout(self, character):
+        if character not in self.layout:
+            raise Exception("Error: Can't get {} from layout {}".format(character, self.layout))
+        return self.shape[self.layout.index(character)]
+
+    @property
+    def width(self):
+        return self.getDimentionByLayout("W")
+
+    @property
+    def height(self):
+        return self.getDimentionByLayout("H")
+
+    @property
+    def channels(self):
+        return self.getDimentionByLayout("C")
+
+    @property
+    def batch(self):
+        return self.getDimentionByLayout("N")
+
+    @property
+    def depth(self):
+        return self.getDimentionByLayout("D")
+
+def get_inputs_info(shape_string, layout_string, batch_size, input_info):
+    shape_map = parse_input_parameters(shape_string, input_info)
+    layout_map = parse_input_parameters(layout_string, input_info)
+    reshape = False
+    info_map = {}
+    for name, descriptor in input_info.items():
+        info = InputInfo()
+        # Precision
+        info.precision = descriptor.precision
+        # Shape
+        if name in shape_map.keys():
+            parsed_shape = [int(dim) for dim in shape_map[name].split(',')]
+            info.shape = parsed_shape
+            reshape = True
+        else:
+            info.shape = descriptor.input_data.shape
+        # Layout
+        info.layout = layout_map[name].upper() if name in layout_map.keys() else descriptor.tensor_desc.layout
+        # Update shape with batch if needed
+        if batch_size != 0:
+            batch_index = info.layout.index('N') if 'N' in info.layout else -1
+            if batch_index != -1 and info.shape[batch_index] != batch_size:
+                info.shape[batch_index] = batch_size
+                reshape = True
+        info_map[name] = info
+    return info_map, reshape
+
+def get_batch_size(inputs_info):
+    batch_size = 0
+    for _, info in inputs_info.items():
+        batch_index = info.layout.index('N') if 'N' in info.layout else -1
+        if batch_index != -1:
+            if batch_size == 0:
+                batch_size = info.shape[batch_index]
+            elif batch_size != info.shape[batch_index]:
+                raise Exception("Can't deterimine batch size: batch is different for different inputs!")
+    if batch_size == 0:
+        batch_size = 1
+    return batch_size
 
 def show_available_devices():
     ie = IECore()