From d3712a148b21cb964c3f99dd92ae7bb3cfae2f2d Mon Sep 17 00:00:00 2001 From: Alexey Lebedev Date: Mon, 14 Feb 2022 20:25:31 +0300 Subject: [PATCH] [tools] cross check tool with api 2.0 (#10058) * save work * save work * save work * basic changes with api 2.0 * Support input file mapping and bin files * Some impovements * remove mapping support * Add -ref_layers parameter * Fix error handler * Update Readme and remove old parameters * Fix readme * remove info about precision * rename layer to op * rename blob to tensor * remove info about shape * remove unused imports --- .../tools/benchmark/utils/inputs_filling.py | 2 +- tools/cross_check_tool/README.md | 12 +- .../cross_check_tool/cross_check_tool.py | 360 +++++++++--------- .../openvino/tools/cross_check_tool/utils.py | 325 +++++++++------- 4 files changed, 365 insertions(+), 334 deletions(-) diff --git a/tools/benchmark_tool/openvino/tools/benchmark/utils/inputs_filling.py b/tools/benchmark_tool/openvino/tools/benchmark/utils/inputs_filling.py index f485ecbde08..5e809a81f80 100644 --- a/tools/benchmark_tool/openvino/tools/benchmark/utils/inputs_filling.py +++ b/tools/benchmark_tool/openvino/tools/benchmark/utils/inputs_filling.py @@ -317,7 +317,7 @@ def parse_path(path, app_input_info): if input_path.exists(): if input_path.is_dir(): input_files += list(str(file_path) for file_path in input_path.iterdir()) - elif input_path.is_file: + elif input_path.is_file(): input_files.append(str(input_path)) else: raise Exception(f"Path '{str(input_path)}' doesn't exist \n {str(input_path)}") diff --git a/tools/cross_check_tool/README.md b/tools/cross_check_tool/README.md index 9eac59954ad..ee1cbb06ba5 100644 --- a/tools/cross_check_tool/README.md +++ b/tools/cross_check_tool/README.md @@ -26,8 +26,6 @@ Model specific arguments: Path to an input image file or multi-input file to infer. Generates input(s) from normal distribution if empty - --batch BATCH, -b BATCH - Overrides batch size. Default is inherited from model --model MODEL, -m MODEL Path to an .xml file that represents the first IR of the trained model to infer. @@ -38,11 +36,11 @@ Model specific arguments: Defines layers to check. Options: all, None - for output layers check, list of comma-separated layer names to check. Default value is None. - --mapping MAPPING, -map MAPPING - Model Optimizer provided mapping for --model/-m - --reference_mapping REFERENCE_MAPPING, -ref_map REFERENCE_MAPPING - Model Optimizer provided mapping for - --reference_model/-ref_model + --ref_layers REFERENCE_LAYERS, -reference_layers REFERENCE_LAYERS + Defines layers to check in reference model. Options: all, None - for + output layers check, list of comma-separated layer + names to check. If not specified the same layers will + be processed as in --layers parameter. --num_of_iterations NUM_OF_ITERATIONS, -ni NUM_OF_ITERATIONS Number of iterations to collect all over the net performance diff --git a/tools/cross_check_tool/openvino/tools/cross_check_tool/cross_check_tool.py b/tools/cross_check_tool/openvino/tools/cross_check_tool/cross_check_tool.py index acd65e5eb0f..58542dcaa7d 100755 --- a/tools/cross_check_tool/openvino/tools/cross_check_tool/cross_check_tool.py +++ b/tools/cross_check_tool/openvino/tools/cross_check_tool/cross_check_tool.py @@ -5,30 +5,22 @@ import datetime import logging as log -import os import sys - +from collections import defaultdict +from typing import Union import numpy as np try: - from openvino import inference_engine as ie - from openvino.inference_engine import IENetwork, IECore + from openvino.runtime import Core, Model, CompiledModel, InferRequest, Output, get_version except Exception as e: exception_type = type(e).__name__ - print(f"The following error happened while importing Python API module:\n[ {exception_type} ] {e}") + print(f"The following error happened while importing OpenVINO Python API module:\n[ {exception_type} ] {e}") sys.exit(1) -try: - import ngraph as ng -except Exception as e: - exception_type = type(e).name - print(f"The following error happened while importing nGraph module:\n[ {exception_type} ] {e}") - sys.exit(1) - -from openvino.tools.cross_check_tool.utils import get_config_dictionary, get_layers_list, print_output_layers, \ +from openvino.tools.cross_check_tool.utils import get_config_dictionary, get_ops_list, print_output_ops, \ input_processing, accuracy_metrics, validate_args, build_parser, set_logger, find_out_cct_mode, \ - print_all_over_the_net_metrics, update_global_accuracy_matrics, blob_counters, performance_metrics, \ - manage_user_outputs_with_mapping, dump_output_file, load_dump, error_handling, print_input_layers, set_verbosity + print_all_over_the_net_metrics, update_global_accuracy_matrics, tensor_counters, performance_metrics, \ + dump_output_file, load_dump, error_handling, print_inputs, set_verbosity, perf_counts_to_dump, load_profiling_info ### @@ -37,22 +29,22 @@ from openvino.tools.cross_check_tool.utils import get_config_dictionary, get_lay @error_handling('plugin of \'{device}\' device config \'{config}\' loading') -def set_plugin_config(core: IECore, device: str, config: str = None): - core.set_config(get_config_dictionary(config_file=config), device_name=device) +def set_plugin_config(core: Core, device: str, config: str = None): + core.set_property(device, get_config_dictionary(config_file=config)) @error_handling('\'{cpu_ext}\' cpu extensions loading') -def set_cpu_extensions(core: IECore, cpu_ext: str): - core.add_extension(cpu_ext, "CPU") +def set_cpu_extensions(core: Core, cpu_ext: str): + core.add_extension(cpu_ext) def get_plugin(device: str, cpu_ext: str = None, config: str = None): - ie = IECore() + core = Core() # log.info('{} plugin:\n API version ............ {}'.format(device, plugin.version), extra={'no_lvl': True}) - set_plugin_config(core=ie, device=device, config=config) + set_plugin_config(core=core, device=device, config=config) if cpu_ext and 'CPU' in device: - set_cpu_extensions(core=ie, cpu_ext=cpu_ext) - return ie + set_cpu_extensions(core=core, cpu_ext=cpu_ext) + return core ### @@ -60,87 +52,108 @@ def get_plugin(device: str, cpu_ext: str = None, config: str = None): ### -@error_handling('reading {model} IR model') -def get_net(model: str, core: IECore): - model_xml = model - model_bin = os.path.splitext(model_xml)[0] + ".bin" - net = core.read_network(model=model_xml, weights=model_bin) - return net +@error_handling('reading {xml_path} IR model') +def get_model(model_path: str, core: Core): + model = core.read_model(model=model_path) + # TODO: can we support it? + if model.is_dynamic(): + raise Exception("Cross check tool doesn't support dynamic models for now.") + return model -@error_handling('loading network to plugin of {device} device') -def get_exec_net(core, net, device): - return core.load_network(network=net, device_name=device) +@error_handling('compiling model for {device} device') +def get_compiled_model(core: Core, model: Model, device: str): + return core.compile_model(model=model, device_name=device) + + +@error_handling('creating infer request') +def get_infer_request(compiled_model: CompiledModel): + return compiled_model.create_infer_request() @error_handling('output \'{output}\' addition for network from model \'{model}\'') -def get_net_copy_with_output(model: str, output: str, core: IECore): - net_copy = get_net(model=model, core=core) - func = ng.function_from_cnn(net_copy) +def get_model_copy_with_output(model: str, output: tuple, core: Core): + model_copy = get_model(model_path=model, core=core) + new_output = None if output not in ['None', None]: - # output with port_id in name is absent in ops list - founded_op = [op for op in func.get_ops() if op.friendly_name == output] - if founded_op: - net_copy.add_outputs(output) - else: - split = output.rsplit(".", 1) - net_copy.add_outputs((split[0], int(split[1]))) - return net_copy + new_output = model_copy.add_outputs(output).pop() + return model_copy, new_output -@error_handling('getting model layers info') -def get_model_info(net: IENetwork): - func = ng.function_from_cnn(net) - ops = func.get_ordered_ops() - return ops, net.input_info, net.outputs +@error_handling('getting model operations info') +def get_model_info(model: Model): + return model.get_ordered_ops(), model.inputs, model.outputs +def check_inputs_and_default_outputs_are_equal(model, ref_model): + if len(model.inputs) != len(ref_model.inputs): + raise Exception("Models have different number of inputs! Cannot cross check!") + if len(model.outputs) != len(ref_model.outputs): + raise Exception("Models have different number of outputs! Cannot cross check!") + for input, ref_input in zip(model.inputs, ref_model.inputs): + if input.any_name != ref_input.any_name: + raise Exception("Models have different inputs! Cannot cross check!") + for output, ref_output in zip(model.outputs, ref_model.outputs): + if output.any_name != ref_output.any_name: + raise Exception("Models have different outputs! Cannot cross check!") + + +def get_ops_intersection(ops, ref_ops): + ops_map = {node.friendly_name: node for node in ops} + operation_names = set(ops_map.keys()) + ref_operation_names = set(node.friendly_name for node in ref_ops) + intersection_names = operation_names.intersection(ref_operation_names) + return [ops_map[intersection_name] for intersection_name in intersection_names] + + +def get_ops_union(ops, ref_ops): + ops_map = {} + for op, ref_op in zip(ops, ref_ops): + ops_map.update({op.friendly_name: op}) + ops_map.update({ref_op.friendly_name: ref_op}) + return ops_map.values() + ### # INFER ### -@error_handling('processing inference') -def get_infer_results(executable_network, inputs: dict): - return executable_network.infer(inputs=inputs) +@error_handling('getting inference results for output: \'{output.any_name}\'') +def get_infer_results(infer_request: InferRequest, output: Output): + return infer_request.get_tensor(output).data -@error_handling('getting performance counts from executable network') -def get_perf_counts(executable_network): - return executable_network.requests[0].get_perf_counts() +@error_handling('getting performance counts from infer request') +def get_profiling_info(infer_request: InferRequest, port: Output): + for pi in infer_request.profiling_info: + if pi.node_name == port.node.friendly_name: + return pi -@error_handling('getting inference results for outputs: \'{output}\' on \'{device}\' device') -def infer(net: IENetwork, core: IECore, device: str, inputs: dict, output: list): - executable_network = get_exec_net(core=core, net=net, device=device) - infer_dict = get_infer_results(executable_network=executable_network, inputs=inputs) - pc = get_perf_counts(executable_network=executable_network) - no_i = 'no_info' - no_info_pc = {'cpu_time': no_i, 'exec_time': no_i, 'layer_type': no_i, 'real_time': no_i, 'status': no_i} - result = {} - for out in output: - if out not in infer_dict: - log.warning(f"There is no '{out}' layer in Inference Engine outputs results") - continue - pc = pc[out] if out in pc else no_info_pc - pc['device'] = device - result = {out: [infer_dict[out], pc]} - return result +@error_handling('processing inference on \'{device}\' device') +def infer(model: Model, core: Core, device: str, inputs: Union[list, dict], output=None): + compiled_model = get_compiled_model(core=core, model=model, device=device) + infer_request = get_infer_request(compiled_model) + infer_request.infer(inputs) + if output: + result = get_infer_results(infer_request, output) + prof_info = get_profiling_info(infer_request, output) + return result, prof_info -@error_handling('getting inference results for outputs: \'{layers}\'') -def overall_accuracy_check(model: str, ref_model: str, out_layers: list, ref_out_layers: list, inputs: dict, - ref_inputs: dict, core: IECore, device: str, ref_core: IECore, ref_device: str, layers: str, +@error_handling('computing overall performance') +def overall_accuracy_check(model: str, ref_model: str, out_ops: list, ref_out_ops: list, inputs: list, + ref_inputs: list, core: Core, device: str, ref_core: Core, ref_device: str, layers: str, num_of_iterations: int): global_times, ref_global_times = [], [] if layers in ['None', None]: - net_copy = get_net_copy_with_output(model=model, output=layers, core=core) - ref_net_copy = get_net_copy_with_output(model=ref_model, output=layers, core=ref_core) + model_copy, _ = get_model_copy_with_output(model=model, output=layers, core=core) + ref_model_copy, _ = get_model_copy_with_output(model=ref_model, output=layers, core=ref_core) for i in range(num_of_iterations): t1 = datetime.datetime.now() - infer(net=net_copy, core=core, device=device, inputs=inputs, output=out_layers) + infer(model=model_copy, core=core, device=device, inputs=inputs) t2 = datetime.datetime.now() - infer(net=ref_net_copy, core=ref_core, device=ref_device, inputs=ref_inputs, output=ref_out_layers) + infer(model=ref_model_copy, core=ref_core, device=ref_device, inputs=ref_inputs) t3 = datetime.datetime.now() global_times.append(t2 - t1) ref_global_times.append(t3 - t2) @@ -149,38 +162,34 @@ def overall_accuracy_check(model: str, ref_model: str, out_layers: list, ref_out def one_ir_mode(args): core = get_plugin(args.device, args.l, args.config) - net = get_net(model=args.model, core=core) - net_layers, net_inputs, net_outputs = get_model_info(net) + model = get_model(model_path=args.model, core=core) + model_ops, model_inputs, model_outputs = get_model_info(model) log.info(f'{args.device} vs {args.reference_device}') log.info(f'The same IR on both devices: {args.model}') - out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers) - print_input_layers(net_inputs) - print_output_layers(out_layers) + out_ops = get_ops_list(model_ops, model_outputs, args.layers) + print_inputs(model_inputs) + print_output_ops(out_ops) ref_core = get_plugin(args.reference_device, args.l, args.reference_config) global_accuracy = [] - inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input) + inputs = input_processing(model_path=args.model, model_inputs=model_inputs, input_file=args.input) global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.model, - out_layers=out_layers, ref_out_layers=out_layers, + out_ops=out_ops, ref_out_ops=out_ops, inputs=inputs, ref_inputs=inputs, core=core, device=args.device, ref_core=ref_core, ref_device=args.reference_device, layers=args.layers, num_of_iterations=args.num_of_iterations) - for out_layer in out_layers: - log.info(f'Layer {out_layer} statistics') - net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core) - results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer]) - if out_layer not in results: - continue - out_blob, pc = results[out_layer] - ref_results = infer(net=net_copy, core=ref_core, device=args.reference_device, - inputs=inputs, output=[out_layer]) - if out_layer not in ref_results: - continue - ref_out_blob, ref_pc = ref_results[out_layer] - a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob) - performance_metrics(pc=pc, ref_pc=ref_pc) - blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob) - global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m) + for op in out_ops: + log.info(f'Layer {op.friendly_name} statistics') + for i in range(op.get_output_size()): + if op.get_output_size() > 1: + log.info(f'Port {i}: ') + model_copy, new_output = get_model_copy_with_output(model=args.model, output=(op.friendly_name, i), core=core) + out_tensor, pc = infer(model=model_copy, core=core, device=args.device, inputs=inputs, output=new_output) + ref_out_tensor, ref_pc = infer(model=model_copy, core=ref_core, device=args.reference_device, inputs=inputs, output=new_output) + a_m = accuracy_metrics(out_tensor, ref_out_tensor) + performance_metrics(args.device, pc, args.reference_device, ref_pc) + tensor_counters(out_tensor, ref_out_tensor) + global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m) print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times, global_accuracy=global_accuracy) @@ -188,108 +197,103 @@ def one_ir_mode(args): def two_ir_mode(args): core = get_plugin(args.device, args.l, args.config) ref_core = get_plugin(args.reference_device, args.l, args.reference_config) - net = get_net(model=args.model, core=core) - net_layers, net_inputs, net_outputs = get_model_info(net) - ref_net = get_net(model=args.reference_model, core=ref_core) - ref_net_layers, ref_net_inputs, ref_net_outputs = get_model_info(ref_net) + model = get_model(model_path=args.model, core=core) + model_ops, model_inputs, model_outputs = get_model_info(model) + ref_model = get_model(model_path=args.reference_model, core=ref_core) + ref_model_ops, _, _ = get_model_info(ref_model) + check_inputs_and_default_outputs_are_equal(model, ref_model) log.info(f'{args.device} vs {args.reference_device}') log.info(f'IR for {args.device} : {args.model}') log.info(f'IR for {args.reference_device} : {args.reference_model}') - out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers) - ref_out_layers = get_layers_list(ref_net_layers, ref_net_inputs, ref_net_outputs, args.layers) - print_input_layers(net_inputs) - print_output_layers(out_layers) - layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping, - user_layers=out_layers) - inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input, - layers_map=layers_map) - ref_inputs = input_processing(model_path=args.reference_model, net_inputs=ref_net_inputs, input_file=args.input, - layers_map=layers_map) + if args.reference_layers: + out_ops = get_ops_list(model_ops, model_outputs, args.layers) + ref_out_ops = get_ops_list(ref_model_ops, model_outputs, args.reference_layers) + if len(out_ops) != len(ref_out_ops): + raise Exception("Number of layers to compare against should be equal!") + else: + ref_out_ops = out_ops = get_ops_list(get_ops_intersection(model_ops, ref_model_ops), model_outputs, args.layers) + print_inputs(model_inputs) + print_output_ops(get_ops_union(out_ops, ref_out_ops)) + inputs = input_processing(model_path=args.model, model_inputs=model_inputs, input_file=args.input) global_accuracy = [] global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.reference_model, - out_layers=out_layers, ref_out_layers=ref_out_layers, - inputs=inputs, ref_inputs=ref_inputs, core=core, + out_ops=out_ops, ref_out_ops=out_ops, + inputs=inputs, ref_inputs=inputs, core=core, device=args.device, ref_core=ref_core, ref_device=args.reference_device, layers=args.layers, num_of_iterations=args.num_of_iterations) - for out_layer in layers_map: - ref_out_layer = layers_map[out_layer] - if out_layer == ref_out_layer: - log.info(f'Layer {out_layer} statistics') + for op, ref_op in zip(out_ops, ref_out_ops): + if op.friendly_name == ref_op.friendly_name: + log.info(f'Layer {op.friendly_name} statistics') else: - log.info(f'Statistics \'{out_layer}\' vs \'{ref_out_layer}\'') - net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core) - ref_net_copy = get_net_copy_with_output(model=args.reference_model, output=ref_out_layer, core=ref_core) - results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer]) - if out_layer not in results: - continue - out_blob, pc = results[out_layer] - ref_results = infer(net=ref_net_copy, core=ref_core, device=args.reference_device, - inputs=ref_inputs, output=[ref_out_layer]) - ref_out_blob, ref_pc = ref_results[ref_out_layer] - if ref_out_layer not in ref_results: - continue - a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob) - performance_metrics(pc=pc, ref_pc=ref_pc) - blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob) - global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m) + if op.get_output_size() != ref_op.get_output_size(): + log.warning(f"Skipping {op.friendly_name} vs {ref_op.frinedly_name} comparison due to different number of outputs!") + continue + log.info(f'Layer {op.friendly_name} vs {ref_op.friendly_name} statistics') + for i in range(op.get_output_size()): + if op.get_output_size() > 1: + log.info(f'Port {i}: ') + model_copy, new_output = get_model_copy_with_output(model=args.model, output=(op.friendly_name, i), core=core) + ref_model_copy, ref_new_output = get_model_copy_with_output(model=args.reference_model, output=(ref_op.friendly_name, i), core=ref_core) + out_tensor, pc = infer(model=model_copy, core=core, device=args.device, inputs=inputs, output=new_output) + ref_out_tensor, ref_pc = infer(model=ref_model_copy, core=ref_core, device=args.reference_device, + inputs=inputs, output=ref_new_output) + a_m = accuracy_metrics(out_tensor, ref_out_tensor) + performance_metrics(args.device, pc, args.reference_device, ref_pc) + tensor_counters(out_tensor, ref_out_tensor) + global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m) print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times, global_accuracy=global_accuracy) def dump_mode(args): core = get_plugin(args.device, args.l, args.config) - net = get_net(model=args.model, core=core) - func = ng.function_from_cnn(net) - ops = func.get_ops() - out_layers = get_layers_list(ops, net.input_info, net.outputs, args.layers) - inputs = input_processing(args.model, net.input_info, args.input) - dump_dict = {} - for out_layer in out_layers: - log.info(f'Layer {out_layer} processing') - net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core) - results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer]) - if out_layer not in results: - continue - out_blob, pc = results[out_layer] - dump_dict[out_layer] = np.array({'blob': out_blob, 'pc': pc}) + model = get_model(model_path=args.model, core=core) + model_ops, model_inputs, model_outputs = get_model_info(model) + out_ops = get_ops_list(model_ops, model_outputs, args.layers) + inputs = input_processing(args.model, model_inputs, args.input) + dump_dict = defaultdict(list) + for op in out_ops: + for i in range(op.get_output_size()): + if op.get_output_size() > 1: + log.info(f'Layer {op.friendly_name}, port {i} processing') + else: + log.info(f'Layer {op.friendly_name} processing') + model_copy, new_output = get_model_copy_with_output(model=args.model, output=(op.friendly_name, i), core=core) + out_tensor, pc = infer(model=model_copy, core=core, device=args.device, inputs=inputs, output=new_output) + dump_dict[op.friendly_name].append(np.array({'tensor': out_tensor, 'pc': perf_counts_to_dump(pc)})) + dump_dict["device"] = args.device dump_output_file(args.model + '_' + args.device + '_dump.npz', dump_dict) def load_mode(args): core = get_plugin(args.device, args.l, args.config) log.info(f'IR for {args.device} : {args.model}') - log.info(f'Loading blob from {args.load}') - net = get_net(model=args.model, core=core) - net_layers, net_inputs, net_outputs = get_model_info(net) - out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers) - print_input_layers(net_inputs) - print_output_layers(out_layers) - layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping, - user_layers=out_layers) - inputs = input_processing(args.model, net_inputs, args.input, layers_map) + log.info(f'Loading tensors from {args.load}') + model = get_model(model_path=args.model, core=core) + model_ops, model_inputs, model_outputs = get_model_info(model) + out_ops = get_ops_list(model_ops, model_outputs, args.layers) + print_inputs(model_inputs) + print_output_ops(out_ops) + inputs = input_processing(args.model, model_inputs, args.input) global_accuracy = [] loaded = load_dump(args.load) - for out_layer in layers_map: - ref_out_layer = layers_map[out_layer] - if out_layer == ref_out_layer: - log.info(f'Layer {out_layer} statistics') + for op in out_ops: + if op.friendly_name in loaded: + log.info(f'Layer {op.friendly_name} statistics') else: - log.info(f'Statistics \'{out_layer}\' vs \'{ref_out_layer}\'') - net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core) - results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer]) - if out_layer not in results: + log.info(f'Statistics for layer \'{op.friendly_name}\' was not dumped. Skipping this layer.') continue - out_blob, pc = results[out_layer] - if ref_out_layer not in loaded: - continue - ref_out_blob = loaded[ref_out_layer]['blob'] - a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob) - if 'pc' in loaded[ref_out_layer]: - ref_pc = loaded[ref_out_layer]['pc'] - performance_metrics(pc=pc, ref_pc=ref_pc) - blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob) - global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m) + for i in range(op.get_output_size()): + if op.get_output_size() > 1: + log.info(f'Port {i}: ') + model_copy, new_output = get_model_copy_with_output(model=args.model, output=(op.friendly_name, i), core=core) + out_tensor, pc = infer(model=model_copy, core=core, device=args.device, inputs=inputs, output=new_output) + ref_out_tensor, ref_pc = loaded[op.friendly_name][i]['tensor'], load_profiling_info(loaded[op.friendly_name][i]['pc']) + a_m = accuracy_metrics(out_tensor, ref_out_tensor) + performance_metrics(args.device, pc, loaded["device"], ref_pc) + tensor_counters(out_tensor, ref_out_tensor) + global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m) print_all_over_the_net_metrics(global_accuracy=global_accuracy) @@ -297,7 +301,7 @@ def main(): set_logger(log.DEBUG) args = validate_args(build_parser().parse_args()) - log.info(f'Inference Engine:\n API version ............ {ie.__version__}', extra={'no_lvl': True}) + log.info(f'OpenVINO:\n API version ............ {get_version()}', extra={'no_lvl': True}) set_verbosity(args.verbosity) mode = find_out_cct_mode(args) if mode == 1: diff --git a/tools/cross_check_tool/openvino/tools/cross_check_tool/utils.py b/tools/cross_check_tool/openvino/tools/cross_check_tool/utils.py index 3abd176f415..21a736954d7 100644 --- a/tools/cross_check_tool/openvino/tools/cross_check_tool/utils.py +++ b/tools/cross_check_tool/openvino/tools/cross_check_tool/utils.py @@ -6,7 +6,10 @@ import logging as log import os import sys import traceback -import xml +from pathlib import Path + +from openvino.runtime import Layout, ProfilingInfo, Output +from openvino.runtime.utils.types import get_dtype try: import cv2 @@ -137,7 +140,7 @@ def build_parser(): '\n--device device_for_model \\' '\n--reference_device reference_device_for_model \n' + '-' * 62 + - '\nFor dumping blob and performance counters run:' + '\nFor dumping tensors and performance counters run:' '\npython3 cross_check_tool.py \\' '\n--input path/to/file/describing/input \\' '\n--model path/to/model/*.xml \\' @@ -162,7 +165,7 @@ def build_parser(): ) model = parser.add_argument_group('Model specific arguments') - model.add_argument('--input', '-i', type=str, action=ExistingFileAction, + model.add_argument('--input', '-i', type=str, help='Path to an input image file or multi-input file to infer. Generates input(s) from normal ' 'distribution if empty') # model.add_argument('--batch', '-b', type=int, help='Overrides batch size. Default is inherited from model') @@ -171,14 +174,13 @@ def build_parser(): model.add_argument('--reference_model', '-ref_m', type=str, action=ExistingFileAction, help='Path to an .xml file that represents the second IR to compare the metrics. ' 'Uses --model if empty') + # TODO: allow to pass layer type model.add_argument('--layers', '-layers', type=str, default=None, help='Defines layers to check. Options: all, None - for output layers check, list of ' 'comma-separated layer names to check. Default value is None.') - model.add_argument('--mapping', '-map', type=str, action=ExistingFileAction, - help='Model Optimizer provided mapping for --model/-m') - model.add_argument('--reference_mapping', '-ref_map', type=str, action=ExistingFileAction, - help='Model Optimizer provided mapping for --reference_model/-ref_model') - + model.add_argument('-ref_layers', '--reference_layers', type=str, default=None, + help='Defines layers to check in referece model. Options: all, None - for output layers check, list of ' + 'comma-separated layer names to check. If not specified the same layers will be processed as in --layers parameter.') plugin = parser.add_argument_group('Plugin specific arguments') plugin.add_argument('--plugin_path', '-pp', type=str, action=ExistingDirAction, help='Path to a plugin folder.') plugin.add_argument('--device', '-d', type=str, required=True, @@ -197,8 +199,8 @@ def build_parser(): modes = parser.add_argument_group('CCT mode arguments') # TODO eps? nobody uses it - modes.add_argument('--dump', help='Enables blobs statistics dumping', action='store_true', default=False) - modes.add_argument('--load', type=str, action=ExistingFileAction, help='Path to a file to load blobs from') + modes.add_argument('--dump', help='Enables tensors statistics dumping', action='store_true', default=False) + modes.add_argument('--load', type=str, action=ExistingFileAction, help='Path to a file to load tensors from') model.add_argument('--num_of_iterations', '-ni', type=int, default=50, help='Number of iterations to collect all over the net performance') parser.add_argument('-v', '--verbosity', action='store_true', default=False, @@ -219,10 +221,10 @@ def validate_args(args): args.model = args.reference_model if args.model == args.reference_model: args.reference_model = None - if args.model != args.reference_model and args.reference_model is not None and args.mapping is None and \ - args.reference_mapping is None: - log.warning('Check over two different IRs was enabled. In case if layer names in this two IRs differ, ' - 'please provide mapping files with --mapping/-map and --reference_mapping/-ref_map') + if args.model != args.reference_model and args.reference_model is not None and (args.layers is None or \ + args.reference_layers is None): + log.warning('Check over two different IRs was enabled. In case if layer names in these two IRs are different, ' + 'please provide both -layers and --reference_layers to compare against.') # device check if args.device is None and args.reference_device is None: raise Exception("Parameters -device/-d and -reference_device/-ref_d are not set. Can not proceed." @@ -273,14 +275,13 @@ def find_out_cct_mode(args): raise Exception('Unknown Cross Check Tool CLI configuration.\nFor more details use -h option') -def print_input_layers(inputs: list): +def print_inputs(inputs: list): word = 'inputs' if len(inputs) > 1 else 'input' - log.info(f"{len(inputs)} {word} detected: {', '.join(inputs)}") + log.info(f"{len(inputs)} {word} detected: {', '.join(input.any_name for input in inputs)}") - -def print_output_layers(outputs: list): - layers = 'layers' if len(outputs) > 1 else 'layer' - log.info(f"Statistics will be dumped for {len(outputs)} {layers}: {', '.join(outputs)}") +def print_output_ops(output_ops: list): + layers = 'layers' if len(output_ops) > 1 else 'layer' + log.info(f"Statistics will be dumped for {len(output_ops)} {layers}: {', '.join(op.friendly_name for op in output_ops)}") ### @@ -306,70 +307,132 @@ def get_config_dictionary(config_file): ### -def read_multi_input_file(input_file: str, net_inputs: dict): +def read_multi_input_file(input_file: str, model_inputs: list): npz = np.load(input_file, allow_pickle=True) files = npz.files - dump = {} - for net_input in net_inputs: - if net_input not in files: - raise Exception(f"Can not find input data for input {net_input} in multi-input file {input_file}.\n" + dump = [] + for model_input in model_inputs: + if model_input.any_name not in files: + raise Exception(f"Can not find input data for input {model_input.any_name} in multi-input file {input_file}.\n" f"Input data was provided for layers: {', '.join(files)}\n" - f"Network inputs: {', '.join(net_inputs.keys())}") - if 'blob' in npz[net_input].item(0): - just_blob = npz[net_input].item(0)['blob'] - network_shape = net_inputs[net_input].input_data.shape - log.info(f'Layer {net_input} shape = {network_shape}, input blob from multi-input file shape = {just_blob.shape}') + f"Network inputs: {', '.join(input.any_name for input in model_inputs)}") + if 'tensor' in npz[model_input.any_name].item(0): + just_tensor = npz[model_input.any_name].item(0)['tensor'] + input_shape = list(model_input.shape) + log.info(f'Layer {model_input.any_name} shape = {input_shape}, input tensor from multi-input file shape = {just_tensor.shape}') try: - reshaped_blob = np.reshape(just_blob, network_shape) + reshaped_tensor = np.reshape(just_tensor, input_shape) except: - raise Exception(f'Can not reshape input blob from multi-input file for layer {net_input} to shape {network_shape}') - dump[net_input] = reshaped_blob + raise Exception(f'Can not reshape input tensor from multi-input file for layer {model_input.any_name} to shape {input_shape}') + dump.append(reshaped_tensor) else: raise Exception( - f'Can not find \'blob\' parameter for input {net_input} in input file {input_file}') + f'Can not find \'tensor\' parameter for input {model_input.any_name} in input file {input_file}') return dump +def is_chw_input(input): + if input.node.layout == Layout("NCHW") or input.node.layout == Layout("CHW"): + return True + shape = input.shape + if len(shape) == 4: + return shape[1] == 3 + if len(shape) == 3: + return shape[0] == 3 + return False + + +def get_input_sizes(input): + if is_chw_input(input): + shape = input.shape + if len(shape) == 3: + return shape[1], shape[2] + else: + return shape[2], shape[3] + else: + shape = input.shape + if len(shape) < 3 or len(shape) > 4: + raise Exception('Can not interpret input shape as image') + if len(shape) == 3: + return shape[0], shape[1] + else: + return shape[1], shape[2] + + @error_handling('reading --input/-i by OpenCV python module. OpenCV version: {}. ' 'It may happen due to wrong input image format'.format(cv2.__version__)) -def read_image_file(input_file: str, net_inputs: dict): - inputs = dict() - if len(net_inputs) == 1: - image = cv2.imread(input_file) - if image is None: - raise Exception('Can not read input image ' + input_file) - only_layer_name = list(net_inputs.keys())[0] - shape = net_inputs[only_layer_name].input_data.shape - if len(shape) != 4: - raise Exception('Can not interpret input shape as image') - n, c, h, w = shape - image = cv2.resize(image, (w, h)) +def read_image_file(image_file: str, model_input: Output): + image_file = str(image_file) + log.info(f'Prepare image {image_file}') + image = cv2.imread(image_file) + if image is None: + raise Exception('Can not read input image ' + image_file) + h, w = get_input_sizes(model_input) + image = cv2.resize(image, (w, h)) + if is_chw_input(model_input): image = image.transpose((2, 0, 1)) # Change data layout from HWC to CHW - image = image.reshape((n, c, h, w)) - inputs[only_layer_name] = image - else: - raise Exception('Multi-input topology detected. Please provide multi-input file to --input key') + if len(model_input.shape) == 4: + image = np.expand_dims(image, 0) # Add batch dimension + return image + + +def get_random_inputs(model_inputs, model_path): + inputs = [np.clip(np.random.normal(0.5, 0.1, size=list(input.shape)), 0, 1) for input in model_inputs] + dump_output_file(model_path + '_random_input_dump.npz', {model_inputs[i].any_name: {'tensor': inputs[i]} for i in range(len(model_inputs))}) return inputs -def input_processing(model_path: str, net_inputs: dict, input_file: str, layers_map: dict = None): - inputs = dict() +def read_binary_file(bin_file, model_input): + log.info(f"Prepare binary file {str(bin_file)}") + binary_file_size = os.path.getsize(bin_file) + tensor_size = model_input.tensor.size + if tensor_size != binary_file_size: + raise Exception(f"File {bin_file} contains {binary_file_size} bytes but model expects {tensor_size}") + return np.reshape(np.fromfile(bin_file, get_dtype(model_input.element_type)), list(model_input.shape)) + + +def input_processing(model_path: str, model_inputs: list, input_file: str): if input_file is None: - for net_input in net_inputs: - inputs[net_input] = np.clip(np.random.normal(0.5, 0.1, size=net_inputs[net_input].input_data.shape), 0, 1) - dump_output_file(model_path + '_random_input_dump.npz', {inp: {'blob': inputs[inp]} for inp in inputs}) - return inputs - try: - inputs = read_multi_input_file(input_file=input_file, net_inputs=net_inputs) - except: - inputs = read_image_file(input_file=input_file, net_inputs=net_inputs) - return inputs + return get_random_inputs(model_inputs, model_path) + else: + inputs = input_file.split(',') + input_names = [input.any_name for input in model_inputs] + input_data = {} + for i in range(min(len(inputs), len(model_inputs))): + splited = inputs[i].rsplit(':', maxsplit=1) + if len(splited) == 0: + raise Exception(f"Can't parse {'input_file'} input parameter!") + tensor_name = None + if len(splited) == 1: + tensor_name = input_names[i] + else: + tensor_name = splited.pop(0) + if tensor_name not in input_names: + raise Exception(f"Input with name {tensor_name} doesn't exist in the model!") + path = Path(splited.pop()) + if path.exists() and path.is_file(): + IMAGE_EXTENSIONS = ['.jpeg', '.jpg', '.png', '.bmp'] + BINARY_EXTENSIONS = ['.bin'] + NUMPY_EXTENSIONS = ['.npy', '.npz'] + current_input = model_inputs[input_names.index(tensor_name)] + if path.suffix.lower() in IMAGE_EXTENSIONS: + input_data[tensor_name] = read_image_file(path, current_input) + elif path.suffix.lower() in BINARY_EXTENSIONS: + input_data[tensor_name] = read_binary_file(path, current_input) + elif path.suffix.lower() in NUMPY_EXTENSIONS: + return read_multi_input_file(path, model_inputs) + elif path.is_dir(): + raise Exception(f"Path `{path}` is a directory! Provide full path to an input file!") + elif not path.exists(): + raise Exception(f"Input file `{path}` doesn't exist!") + return input_data -def accuracy_metrics(out_blob, ref_out_blob): - if out_blob.size != ref_out_blob.size: - raise Exception(f'Different number of elements in blobs {out_blob.size} and {ref_out_blob.size}. Can not compare') - abs_diff = np.absolute(out_blob - ref_out_blob) +def accuracy_metrics(tensor, ref_tensor): + if tensor.size != ref_tensor.size: + raise Exception(f'Different number of elements in tensors {tensor.size} and {ref_tensor.size}. Can not compare') + abs_diff = np.absolute(tensor - ref_tensor) + np.seterr(divide='ignore', invalid='ignore') rel_diff = np.divide(abs_diff, np.min(abs_diff) if np.min(abs_diff) != 0 else 1e-20) metrics = [ @@ -377,14 +440,14 @@ def accuracy_metrics(out_blob, ref_out_blob): ('Min absolute difference', np.min(abs_diff)), ('Max relative difference', np.max(rel_diff)), ('Min relative difference', np.min(rel_diff)), - ('Min reference value', np.min(ref_out_blob)), - ('Min absolute reference value', np.min(np.abs(ref_out_blob))), - ('Max reference value', np.max(ref_out_blob)), - ('Max absolute reference value', np.max(np.abs(ref_out_blob))), - ('Min actual value', np.min(out_blob)), - ('Min absolute actual value', np.min(np.abs(out_blob))), - ('Max actual value', np.max(out_blob)), - ('Max absolute actual value', np.max(np.abs(out_blob))) + ('Min reference value', np.min(ref_tensor)), + ('Min absolute reference value', np.min(np.abs(ref_tensor))), + ('Max reference value', np.max(ref_tensor)), + ('Max absolute reference value', np.max(np.abs(ref_tensor))), + ('Min actual value', np.min(tensor)), + ('Min absolute actual value', np.min(np.abs(tensor))), + ('Max actual value', np.max(tensor)), + ('Max absolute actual value', np.max(np.abs(tensor))) ] for key, value in metrics: @@ -395,24 +458,24 @@ def accuracy_metrics(out_blob, ref_out_blob): return {metric: value for metric, value in metrics} -def performance_metrics(pc, ref_pc): +def performance_metrics(device, pc, ref_device, ref_pc): compare = [ - ('Device', '-d ' + pc['device'], '-ref_d ' + ref_pc['device']), - ('Status', pc['status'], ref_pc['status']), - ('Layer type', pc['layer_type'], ref_pc['layer_type']), - ('Real time, microsec', pc['real_time'], ref_pc['real_time']) + ('Device', '-d ' + device, '-ref_d ' + ref_device), + ('Status', str(pc.status), str(ref_pc.status)), + ('Layer type', pc.node_type, ref_pc.node_type), + ('Real time, microsec', str(pc.real_time), str(ref_pc.real_time)) ] for metric, actual, reference in compare: log.info(f'{metric:>35}: {actual:>16} {reference:>16}', extra={'no_lvl': True}) -def blob_counters(out_blob, ref_out_blob): +def tensor_counters(tensor, ref_tensor): counters = [ - ('Number of NAN', np.sum(np.isnan(out_blob)), np.sum(np.isnan(ref_out_blob))), - ('Number of INF', np.sum(np.isinf(out_blob)), np.sum(np.isinf(ref_out_blob))), - ('Number of ZERO', out_blob.size - np.count_nonzero(out_blob), - ref_out_blob.size - np.count_nonzero(ref_out_blob)) + ('Number of NAN', np.sum(np.isnan(tensor)), np.sum(np.isnan(ref_tensor))), + ('Number of INF', np.sum(np.isinf(tensor)), np.sum(np.isinf(ref_tensor))), + ('Number of ZERO', tensor.size - np.count_nonzero(tensor), + ref_tensor.size - np.count_nonzero(ref_tensor)) ] for metric, actual, reference in counters: log.info(f'{metric:>35}: {actual:>16} {reference:>16}', extra={'no_lvl': True}) @@ -435,7 +498,7 @@ def update_global_accuracy_matrics(global_accuracy: list, current_accuracy: dict return global_accuracy -def print_all_over_the_net_metrics(global_accuracy: (str, float), global_times: list = None, +def print_all_over_the_net_metrics(global_accuracy: list, global_times: list = None, ref_global_times: list = None): if global_times is not None and ref_global_times is not None and len(global_times) and len(ref_global_times): log.info('-' * 70, extra={'no_lvl': True}) @@ -453,74 +516,39 @@ def print_all_over_the_net_metrics(global_accuracy: (str, float), global_times: ### -def read_mapping(file_name: str): - # TODO check twice - mapping_dict = {} - xml_tree = xml.etree.ElementTree.parse(file_name) - xml_root = xml_tree.getroot() - for child in xml_root: - fw_info = child.find('.//framework') - ir_info = child.find('.//IR') - if fw_info is None: - continue - if ir_info is None: - continue - framework_name = fw_info.attrib['name'] + ':' + fw_info.attrib['out_port_id'] - ir_name = ir_info.attrib['name'] if ir_info is not None else None - ir_layer_id = int(ir_info.attrib['id']) if ir_info is not None else None - mapping_dict[framework_name] = (ir_name, ir_layer_id) - return mapping_dict - - -def map_layers(mapping_file: str = None, ref_mapping_file: str = None): - if mapping_file is not None and ref_mapping_file is not None: - mapping = read_mapping(mapping_file) - ref_mapping = read_mapping(ref_mapping_file) - mapping = {layer: ref_layer for layer in mapping for ref_layer in ref_mapping if layer == ref_layer} - return mapping - - -def manage_user_outputs_with_mapping(mapping, reference_mapping, user_layers): - if mapping is not None and reference_mapping is not None: - layers_map = map_layers(mapping, reference_mapping) - else: - layers_map = {layer: layer for layer in user_layers} - for layer in user_layers: - if layer not in layers_map: - if mapping is not None and reference_mapping is not None: - log.warning( - f'Can not map layer {layer} from --model/-m to any layer from --reference_model/-ref_m') - else: - log.warning(f'Can not find layer {layer} in --reference_model/-ref_m model') - for layer in layers_map: - if layer not in user_layers: - del layers_map[layer] - return layers_map - - -def get_layers_list(all_layers: list, inputs: dict, outputs: list, layers: str): +def get_ops_list(all_ops: list, outputs: list, layers: str): if layers is not None and layers != 'None': if layers == 'all': - return {layer.get_friendly_name(): layer for layer in all_layers \ - if layer.get_type_name() not in ['Constant', 'Result']} + return [op for op in all_ops if op.get_type_name() not in ['Constant', 'Result', 'Parameter']] else: - all_layers_names = {op.get_friendly_name() : op for op in all_layers} - user_layers = [layer.strip() for layer in layers.split(',')] - layers_to_check = [] - for user_layer in user_layers: - if user_layer not in all_layers_names: - raise Exception(f"Layer {user_layer} doesn't exist in the model") - if user_layer in inputs: - raise Exception(f"Layer {user_layer} is input layer. Can not proceed") - if all_layers_names[user_layer].get_type_name() != 'Result': - layers_to_check.append(user_layer) + all_ops_map = {op.friendly_name: op for op in all_ops} + user_ops = [layer.strip() for layer in layers.split(',')] + new_outputs = [] + for user_op in user_ops: + if user_op not in all_ops_map: + raise Exception(f"Operation with name `{user_op}` doesn't exist in the model") else: - # if layer type is Result - add previous layer - prev_layer = all_layers[len(all_layers)-2] - layers_to_check.append(prev_layer.get_friendly_name()) - return layers_to_check + new_outputs.append(all_ops_map[user_op]) + return new_outputs else: - return outputs + return [output.node for output in outputs] + + +def perf_counts_to_dump(prof_info): + perf_count = {} + perf_count["status"] = prof_info.status + perf_count["real_time"] = prof_info.real_time + perf_count["cpu_time"] = prof_info.cpu_time + perf_count["exec_type"] = prof_info.exec_type + perf_count["node_type"] = prof_info.node_type + return perf_count + + +def load_profiling_info(pi_dumped): + prof_info = ProfilingInfo() + for property, value in pi_dumped.items(): + setattr(prof_info, property, value) + return prof_info ### @@ -534,5 +562,6 @@ def dump_output_file(output_file, dump_dict): def load_dump(file_to_load: str): npz = np.load(file_to_load, allow_pickle=True) - dump = {file: npz[file].item(0) for file in npz} + dump = {file: [npz[file].item(i).item(0) for i in range(npz[file].size)] for file in npz if file != "device"} + dump["device"] = npz["device"].item(0) return dump