[GPU] Add separate config for each device (#7421)

tools/benchmark_tool/openvino/tools/benchmark/README.md

@@ -76,7 +76,8 @@ Options:
                         value is CPU. Use '-d HETERO:<comma separated devices
                         list>' format to specify HETERO plugin. Use '-d
                         MULTI:<comma separated devices list>' format to
-                        specify MULTI plugin. The application looks for a
+                        specify MULTI plugin. Use "-d GPU.X" format to specify
+                        device id for GPU devices. The application looks for a
                         suitable plugin for the specified device.
   -l PATH_TO_EXTENSION, --path_to_extension PATH_TO_EXTENSION
                         Optional. Required for CPU custom layers. Absolute

tools/benchmark_tool/openvino/tools/benchmark/main.py

@@ -83,6 +83,22 @@ def run(args):
 
         # --------------------- 3. Setting device configuration --------------------------------------------------------
         next_step()
+        def get_device_type_from_name(name) :
+            new_name = str(name)
+            new_name = new_name.split(".", 1)[0]
+            new_name = new_name.split("(", 1)[0]
+            return new_name
+
+        ## Set default values from dumped config
+        default_devices = set()
+        for device in devices:
+            device_type = get_device_type_from_name(device)
+            if device_type in config and device not in config:
+                config[device] = config[device_type].copy()
+                default_devices.add(device_type)
+
+        for def_device in default_devices:
+            config.pop(def_device)
 
         perf_counts = False
         for device in devices:
@@ -115,7 +131,7 @@ def run(args):
                     config[device]['PERFORMANCE_HINT_NUM_REQUESTS'] = str(args.number_infer_requests)
             ## the rest are individual per-device settings (overriding the values the device will deduce from perf hint)
             def set_throughput_streams():
-                key = device + "_THROUGHPUT_STREAMS"
+                key = get_device_type_from_name(device) + "_THROUGHPUT_STREAMS"
                 if device in device_number_streams.keys():
                     ## set to user defined value
                     supported_config_keys = benchmark.ie.get_metric(device, 'SUPPORTED_CONFIG_KEYS')
@@ -129,11 +145,11 @@ def run(args):
                                    "Although the automatic selection usually provides a reasonable performance, "
                                    "but it still may be non-optimal for some cases, for more information look at README.")
                     if device != MYRIAD_DEVICE_NAME:  ## MYRIAD sets the default number of streams implicitly
-                        config[device][key] = device + "_THROUGHPUT_AUTO"
+                        config[device][key] = get_device_type_from_name(device) + "_THROUGHPUT_AUTO"
                 if key in config[device].keys():
                     device_number_streams[device] = config[device][key]
 
-            if device == CPU_DEVICE_NAME: # CPU supports few special performance-oriented keys
+            if CPU_DEVICE_NAME in device: # CPU supports few special performance-oriented keys
                 # limit threading for CPU portion of inference
                 if args.number_threads and is_flag_set_in_command_line("nthreads"):
                     config[device]['CPU_THREADS_NUM'] = str(args.number_threads)
@@ -152,7 +168,7 @@ def run(args):
 
                 ## for CPU execution, more throughput-oriented execution via streams
                 set_throughput_streams()
-            elif device == GPU_DEVICE_NAME:
+            elif GPU_DEVICE_NAME in device:
                 ## for GPU execution, more throughput-oriented execution via streams
                 set_throughput_streams()
 
@@ -160,10 +176,10 @@ def run(args):
                     logger.warning("Turn on GPU throttling. Multi-device execution with the CPU + GPU performs best with GPU throttling hint, " +
                                    "which releases another CPU thread (that is otherwise used by the GPU driver for active polling)")
                     config[device]['GPU_PLUGIN_THROTTLE'] = '1'
-            elif device == MYRIAD_DEVICE_NAME:
+            elif MYRIAD_DEVICE_NAME in device:
                 set_throughput_streams()
                 config[device]['LOG_LEVEL'] = 'LOG_INFO'
-            elif device == GNA_DEVICE_NAME:
+            elif GNA_DEVICE_NAME in device:
                 if is_flag_set_in_command_line('qb'):
                     if args.qb == 8:
                         config[device]['GNA_PRECISION'] = 'I8'
@@ -303,8 +319,8 @@ def run(args):
 
         # Update number of streams
         for device in device_number_streams.keys():
-            key = device + '_THROUGHPUT_STREAMS'
-            device_number_streams[device] = exe_network.get_config(key)
+            key = get_device_type_from_name(device) + '_THROUGHPUT_STREAMS'
+            device_number_streams[device] = benchmark.ie.get_config(device, key)
 
         # Number of requests
         infer_requests = exe_network.requests
@@ -342,7 +358,7 @@ def run(args):
 
         # ------------------------------------ 10. Measuring performance -----------------------------------------------
 
-        output_string = process_help_inference_string(benchmark, exe_network)
+        output_string = process_help_inference_string(benchmark, device_number_streams)
 
         next_step(additional_info=output_string)
         progress_bar_total_count = 10000

tools/benchmark_tool/openvino/tools/benchmark/utils/utils.py

@@ -165,8 +165,7 @@ def parse_devices(device_string):
     devices = device_string
     if ':' in devices:
         devices = devices.partition(':')[2]
-    return [d[:d.index('(')] if '(' in d else
-            d[:d.index('.')] if '.' in d else d for d in devices.split(',')]
+    return [d for d in devices.split(',')]
 
 
 def parse_nstreams_value_per_device(devices, values_string):
@@ -194,19 +193,15 @@ def parse_nstreams_value_per_device(devices, values_string):
     return result
 
 
-def process_help_inference_string(benchmark_app, exe_network):
+def process_help_inference_string(benchmark_app, device_number_streams):
     output_string = f'Start inference {benchmark_app.api_type}hronously'
     if benchmark_app.api_type == 'async':
         output_string += f', {benchmark_app.nireq} inference requests'
 
         device_ss = ''
-        if CPU_DEVICE_NAME in benchmark_app.device:
-            device_ss += str(exe_network.get_config('CPU_THROUGHPUT_STREAMS'))
-            device_ss += f' streams for {CPU_DEVICE_NAME}'
-        if GPU_DEVICE_NAME in benchmark_app.device:
+        for device, streams in device_number_streams.items():
             device_ss += ', ' if device_ss else ''
-            device_ss += str(exe_network.get_config('GPU_THROUGHPUT_STREAMS'))
-            device_ss += f' streams for {GPU_DEVICE_NAME}'
+            device_ss += f'{streams} streams for {device}'
 
         if device_ss:
             output_string += ' using ' + device_ss