From 7c56155dae77b68c8ac60b4a73cfaa2c70c43b92 Mon Sep 17 00:00:00 2001 From: Julia Kamelina Date: Wed, 3 Nov 2021 01:55:27 +0300 Subject: [PATCH] [Python Tools] Align namespace for cross_check_tool (#7786) * align openvino namespace * update README --- tools/cross_check_tool/README.md | 7 + tools/cross_check_tool/cross_check_tool.py | 312 +---------------- tools/cross_check_tool/openvino/__init__.py | 3 + .../{ => openvino/tools}/__init__.py | 0 .../tools/cross_check_tool/__init__.py | 3 + .../cross_check_tool/cross_check_tool.py | 319 ++++++++++++++++++ .../tools/cross_check_tool}/utils.py | 0 tools/cross_check_tool/setup.py | 40 +++ 8 files changed, 375 insertions(+), 309 deletions(-) create mode 100644 tools/cross_check_tool/openvino/__init__.py rename tools/cross_check_tool/{ => openvino/tools}/__init__.py (100%) create mode 100644 tools/cross_check_tool/openvino/tools/cross_check_tool/__init__.py create mode 100755 tools/cross_check_tool/openvino/tools/cross_check_tool/cross_check_tool.py rename tools/cross_check_tool/{ => openvino/tools/cross_check_tool}/utils.py (100%) create mode 100644 tools/cross_check_tool/setup.py diff --git a/tools/cross_check_tool/README.md b/tools/cross_check_tool/README.md index a2f37929e4b..9eac59954ad 100644 --- a/tools/cross_check_tool/README.md +++ b/tools/cross_check_tool/README.md @@ -72,6 +72,13 @@ CCT mode arguments: --load LOAD Path to a file to load blobs from ``` + +Cross Check Tool can also be installed via: +```sh +$python3 -m pip install /tools/cross_check_tool +``` +In this case, to run the tool, call `cross_check_tool` on the command line with necessary parameters. + ### Examples 1. To check per-layer accuracy and performance of inference in FP32 precision on the CPU against the GPU, run: diff --git a/tools/cross_check_tool/cross_check_tool.py b/tools/cross_check_tool/cross_check_tool.py index c2d504e8e33..4a16e8fbb87 100755 --- a/tools/cross_check_tool/cross_check_tool.py +++ b/tools/cross_check_tool/cross_check_tool.py @@ -3,315 +3,9 @@ # Copyright (C) 2018-2021 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import datetime -import logging as log -import os import sys - -import numpy as np - -try: - from openvino import inference_engine as ie - from openvino.inference_engine import IENetwork, IECore -except Exception as e: - exception_type = type(e).__name__ - print(f"The following error happened while importing Python API module:\n[ {exception_type} ] {e}") - sys.exit(1) - -try: - import ngraph as ng -except Exception as e: - exception_type = type(e).name - print(f"The following error happened while importing nGraph module:\n[ {exception_type} ] {e}") - sys.exit(1) - -from utils import get_config_dictionary, get_layers_list, print_output_layers, input_processing, \ - accuracy_metrics, validate_args, build_parser, set_logger, find_out_cct_mode, print_all_over_the_net_metrics, \ - update_global_accuracy_matrics, blob_counters, performance_metrics, manage_user_outputs_with_mapping, \ - dump_output_file, load_dump, error_handling, print_input_layers, set_verbosity +from openvino.tools.cross_check_tool.cross_check_tool import main -### -# PLUGIN -### - - -@error_handling('plugin of \'{device}\' device config \'{config}\' loading') -def set_plugin_config(core: IECore, device: str, config: str = None): - core.set_config(get_config_dictionary(config_file=config), device_name=device) - - -@error_handling('\'{cpu_ext}\' cpu extensions loading') -def set_cpu_extensions(core: IECore, cpu_ext: str): - core.add_extension(cpu_ext, "CPU") - - -def get_plugin(device: str, cpu_ext: str = None, config: str = None): - ie = IECore() - # log.info('{} plugin:\n API version ............ {}'.format(device, plugin.version), extra={'no_lvl': True}) - set_plugin_config(core=ie, device=device, config=config) - if cpu_ext and 'CPU' in device: - set_cpu_extensions(core=ie, cpu_ext=cpu_ext) - return ie - - -### -# MODEL -### - - -@error_handling('reading {model} IR model') -def get_net(model: str, core: IECore): - model_xml = model - model_bin = os.path.splitext(model_xml)[0] + ".bin" - net = core.read_network(model=model_xml, weights=model_bin) - return net - - -@error_handling('loading network to plugin of {device} device') -def get_exec_net(core, net, device): - return core.load_network(network=net, device_name=device) - - -@error_handling('output \'{output}\' addition for network from model \'{model}\'') -def get_net_copy_with_output(model: str, output: str, core: IECore): - net_copy = get_net(model=model, core=core) - func = ng.function_from_cnn(net_copy) - if output not in ['None', None]: - # output with port_id in name is absent in ops list - founded_op = [op for op in func.get_ops() if op.friendly_name == output] - if founded_op: - net_copy.add_outputs(output) - else: - split = output.rsplit(".", 1) - net_copy.add_outputs((split[0], int(split[1]))) - return net_copy - - -@error_handling('getting model layers info') -def get_model_info(net: IENetwork): - func = ng.function_from_cnn(net) - ops = func.get_ordered_ops() - return ops, net.input_info, net.outputs - - -### -# INFER -### - - -@error_handling('processing inference') -def get_infer_results(executable_network, inputs: dict): - return executable_network.infer(inputs=inputs) - - -@error_handling('getting performance counts from executable network') -def get_perf_counts(executable_network): - return executable_network.requests[0].get_perf_counts() - - -@error_handling('getting inference results for outputs: \'{output}\' on \'{device}\' device') -def infer(net: IENetwork, core: IECore, device: str, inputs: dict, output: list): - executable_network = get_exec_net(core=core, net=net, device=device) - infer_dict = get_infer_results(executable_network=executable_network, inputs=inputs) - pc = get_perf_counts(executable_network=executable_network) - no_i = 'no_info' - no_info_pc = {'cpu_time': no_i, 'exec_time': no_i, 'layer_type': no_i, 'real_time': no_i, 'status': no_i} - result = {} - for out in output: - if out not in infer_dict: - log.warning(f"There is no '{out}' layer in Inference Engine outputs results") - continue - pc = pc[out] if out in pc else no_info_pc - pc['device'] = device - result = {out: [infer_dict[out], pc]} - return result - - -@error_handling('getting inference results for outputs: \'{layers}\'') -def overall_accuracy_check(model: str, ref_model: str, out_layers: list, ref_out_layers: list, inputs: dict, - ref_inputs: dict, core: IECore, device: str, ref_core: IECore, ref_device: str, layers: str, - num_of_iterations: int): - global_times, ref_global_times = [], [] - if layers in ['None', None]: - net_copy = get_net_copy_with_output(model=model, output=layers, core=core) - ref_net_copy = get_net_copy_with_output(model=ref_model, output=layers, core=ref_core) - for i in range(num_of_iterations): - t1 = datetime.datetime.now() - infer(net=net_copy, core=core, device=device, inputs=inputs, output=out_layers) - t2 = datetime.datetime.now() - infer(net=ref_net_copy, core=ref_core, device=ref_device, inputs=ref_inputs, output=ref_out_layers) - t3 = datetime.datetime.now() - global_times.append(t2 - t1) - ref_global_times.append(t3 - t2) - return global_times, ref_global_times - - -def one_ir_mode(args): - core = get_plugin(args.device, args.l, args.config) - net = get_net(model=args.model, core=core) - net_layers, net_inputs, net_outputs = get_model_info(net) - log.info(f'{args.device} vs {args.reference_device}') - log.info(f'The same IR on both devices: {args.model}') - out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers) - print_input_layers(net_inputs) - print_output_layers(out_layers) - ref_core = get_plugin(args.reference_device, args.l, args.reference_config) - global_accuracy = [] - inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input) - global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.model, - out_layers=out_layers, ref_out_layers=out_layers, - inputs=inputs, ref_inputs=inputs, core=core, - device=args.device, ref_core=ref_core, - ref_device=args.reference_device, layers=args.layers, - num_of_iterations=args.num_of_iterations) - for out_layer in out_layers: - log.info(f'Layer {out_layer} statistics') - net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core) - results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer]) - if out_layer not in results: - continue - out_blob, pc = results[out_layer] - ref_results = infer(net=net_copy, core=ref_core, device=args.reference_device, - inputs=inputs, output=[out_layer]) - if out_layer not in ref_results: - continue - ref_out_blob, ref_pc = ref_results[out_layer] - a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob) - performance_metrics(pc=pc, ref_pc=ref_pc) - blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob) - global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m) - print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times, - global_accuracy=global_accuracy) - - -def two_ir_mode(args): - core = get_plugin(args.device, args.l, args.config) - ref_core = get_plugin(args.reference_device, args.l, args.reference_config) - net = get_net(model=args.model, core=core) - net_layers, net_inputs, net_outputs = get_model_info(net) - ref_net = get_net(model=args.reference_model, core=ref_core) - ref_net_layers, ref_net_inputs, ref_net_outputs = get_model_info(ref_net) - log.info(f'{args.device} vs {args.reference_device}') - log.info(f'IR for {args.device} : {args.model}') - log.info(f'IR for {args.reference_device} : {args.reference_model}') - out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers) - ref_out_layers = get_layers_list(ref_net_layers, ref_net_inputs, ref_net_outputs, args.layers) - print_input_layers(net_inputs) - print_output_layers(out_layers) - layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping, - user_layers=out_layers) - inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input, - layers_map=layers_map) - ref_inputs = input_processing(model_path=args.reference_model, net_inputs=ref_net_inputs, input_file=args.input, - layers_map=layers_map) - global_accuracy = [] - global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.reference_model, - out_layers=out_layers, ref_out_layers=ref_out_layers, - inputs=inputs, ref_inputs=ref_inputs, core=core, - device=args.device, ref_core=ref_core, - ref_device=args.reference_device, layers=args.layers, - num_of_iterations=args.num_of_iterations) - for out_layer in layers_map: - ref_out_layer = layers_map[out_layer] - if out_layer == ref_out_layer: - log.info(f'Layer {out_layer} statistics') - else: - log.info(f'Statistics \'{out_layer}\' vs \'{ref_out_layer}\'') - net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core) - ref_net_copy = get_net_copy_with_output(model=args.reference_model, output=ref_out_layer, core=ref_core) - results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer]) - if out_layer not in results: - continue - out_blob, pc = results[out_layer] - ref_results = infer(net=ref_net_copy, core=ref_core, device=args.reference_device, - inputs=ref_inputs, output=[ref_out_layer]) - ref_out_blob, ref_pc = ref_results[ref_out_layer] - if ref_out_layer not in ref_results: - continue - a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob) - performance_metrics(pc=pc, ref_pc=ref_pc) - blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob) - global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m) - print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times, - global_accuracy=global_accuracy) - - -def dump_mode(args): - core = get_plugin(args.device, args.l, args.config) - net = get_net(model=args.model, core=core) - func = ng.function_from_cnn(net) - ops = func.get_ops() - out_layers = get_layers_list(ops, net.input_info, net.outputs, args.layers) - inputs = input_processing(args.model, net.input_info, args.input) - dump_dict = {} - for out_layer in out_layers: - log.info(f'Layer {out_layer} processing') - net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core) - results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer]) - if out_layer not in results: - continue - out_blob, pc = results[out_layer] - dump_dict[out_layer] = np.array({'blob': out_blob, 'pc': pc}) - dump_output_file(args.model + '_' + args.device + '_dump.npz', dump_dict) - - -def load_mode(args): - core = get_plugin(args.device, args.l, args.config) - log.info(f'IR for {args.device} : {args.model}') - log.info(f'Loading blob from {args.load}') - net = get_net(model=args.model, core=core) - net_layers, net_inputs, net_outputs = get_model_info(net) - out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers) - print_input_layers(net_inputs) - print_output_layers(out_layers) - layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping, - user_layers=out_layers) - inputs = input_processing(args.model, net_inputs, args.input, layers_map) - global_accuracy = [] - loaded = load_dump(args.load) - for out_layer in layers_map: - ref_out_layer = layers_map[out_layer] - if out_layer == ref_out_layer: - log.info(f'Layer {out_layer} statistics') - else: - log.info(f'Statistics \'{out_layer}\' vs \'{ref_out_layer}\'') - net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core) - results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer]) - if out_layer not in results: - continue - out_blob, pc = results[out_layer] - if ref_out_layer not in loaded: - continue - ref_out_blob = loaded[ref_out_layer]['blob'] - a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob) - if 'pc' in loaded[ref_out_layer]: - ref_pc = loaded[ref_out_layer]['pc'] - performance_metrics(pc=pc, ref_pc=ref_pc) - blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob) - global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m) - print_all_over_the_net_metrics(global_accuracy=global_accuracy) - - -def main(args): - log.info(f'Inference Engine:\n API version ............ {ie.__version__}', extra={'no_lvl': True}) - set_verbosity(args.verbosity) - mode = find_out_cct_mode(args) - if mode == 1: - log.info('Cross check with one IR was enabled') - one_ir_mode(args) - elif mode == 2: - log.info('Cross check with two IRs was enabled') - two_ir_mode(args) - elif mode == 3: - log.info('Dump mode was enabled') - dump_mode(args) - elif mode == 4: - log.info('Load mode was enabled') - load_mode(args) - log.info("Execution successful") - - -if __name__ == '__main__': - set_logger(log.DEBUG) - main(validate_args(build_parser().parse_args())) +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/tools/cross_check_tool/openvino/__init__.py b/tools/cross_check_tool/openvino/__init__.py new file mode 100644 index 00000000000..9dcdeb46b4d --- /dev/null +++ b/tools/cross_check_tool/openvino/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/tools/cross_check_tool/__init__.py b/tools/cross_check_tool/openvino/tools/__init__.py similarity index 100% rename from tools/cross_check_tool/__init__.py rename to tools/cross_check_tool/openvino/tools/__init__.py diff --git a/tools/cross_check_tool/openvino/tools/cross_check_tool/__init__.py b/tools/cross_check_tool/openvino/tools/cross_check_tool/__init__.py new file mode 100644 index 00000000000..60324954ef5 --- /dev/null +++ b/tools/cross_check_tool/openvino/tools/cross_check_tool/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + diff --git a/tools/cross_check_tool/openvino/tools/cross_check_tool/cross_check_tool.py b/tools/cross_check_tool/openvino/tools/cross_check_tool/cross_check_tool.py new file mode 100755 index 00000000000..5a147eb460b --- /dev/null +++ b/tools/cross_check_tool/openvino/tools/cross_check_tool/cross_check_tool.py @@ -0,0 +1,319 @@ +#!/usr/bin/python3 + +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import datetime +import logging as log +import os +import sys + +import numpy as np + +try: + from openvino import inference_engine as ie + from openvino.inference_engine import IENetwork, IECore +except Exception as e: + exception_type = type(e).__name__ + print(f"The following error happened while importing Python API module:\n[ {exception_type} ] {e}") + sys.exit(1) + +try: + import ngraph as ng +except Exception as e: + exception_type = type(e).name + print(f"The following error happened while importing nGraph module:\n[ {exception_type} ] {e}") + sys.exit(1) + +from openvino.tools.cross_check_tool.utils import get_config_dictionary, get_layers_list, print_output_layers, \ + input_processing, accuracy_metrics, validate_args, build_parser, set_logger, find_out_cct_mode, \ + print_all_over_the_net_metrics, update_global_accuracy_matrics, blob_counters, performance_metrics, \ + manage_user_outputs_with_mapping, dump_output_file, load_dump, error_handling, print_input_layers, set_verbosity + + +### +# PLUGIN +### + + +@error_handling('plugin of \'{device}\' device config \'{config}\' loading') +def set_plugin_config(core: IECore, device: str, config: str = None): + core.set_config(get_config_dictionary(config_file=config), device_name=device) + + +@error_handling('\'{cpu_ext}\' cpu extensions loading') +def set_cpu_extensions(core: IECore, cpu_ext: str): + core.add_extension(cpu_ext, "CPU") + + +def get_plugin(device: str, cpu_ext: str = None, config: str = None): + ie = IECore() + # log.info('{} plugin:\n API version ............ {}'.format(device, plugin.version), extra={'no_lvl': True}) + set_plugin_config(core=ie, device=device, config=config) + if cpu_ext and 'CPU' in device: + set_cpu_extensions(core=ie, cpu_ext=cpu_ext) + return ie + + +### +# MODEL +### + + +@error_handling('reading {model} IR model') +def get_net(model: str, core: IECore): + model_xml = model + model_bin = os.path.splitext(model_xml)[0] + ".bin" + net = core.read_network(model=model_xml, weights=model_bin) + return net + + +@error_handling('loading network to plugin of {device} device') +def get_exec_net(core, net, device): + return core.load_network(network=net, device_name=device) + + +@error_handling('output \'{output}\' addition for network from model \'{model}\'') +def get_net_copy_with_output(model: str, output: str, core: IECore): + net_copy = get_net(model=model, core=core) + func = ng.function_from_cnn(net_copy) + if output not in ['None', None]: + # output with port_id in name is absent in ops list + founded_op = [op for op in func.get_ops() if op.friendly_name == output] + if founded_op: + net_copy.add_outputs(output) + else: + split = output.rsplit(".", 1) + net_copy.add_outputs((split[0], int(split[1]))) + return net_copy + + +@error_handling('getting model layers info') +def get_model_info(net: IENetwork): + func = ng.function_from_cnn(net) + ops = func.get_ordered_ops() + return ops, net.input_info, net.outputs + + +### +# INFER +### + + +@error_handling('processing inference') +def get_infer_results(executable_network, inputs: dict): + return executable_network.infer(inputs=inputs) + + +@error_handling('getting performance counts from executable network') +def get_perf_counts(executable_network): + return executable_network.requests[0].get_perf_counts() + + +@error_handling('getting inference results for outputs: \'{output}\' on \'{device}\' device') +def infer(net: IENetwork, core: IECore, device: str, inputs: dict, output: list): + executable_network = get_exec_net(core=core, net=net, device=device) + infer_dict = get_infer_results(executable_network=executable_network, inputs=inputs) + pc = get_perf_counts(executable_network=executable_network) + no_i = 'no_info' + no_info_pc = {'cpu_time': no_i, 'exec_time': no_i, 'layer_type': no_i, 'real_time': no_i, 'status': no_i} + result = {} + for out in output: + if out not in infer_dict: + log.warning(f"There is no '{out}' layer in Inference Engine outputs results") + continue + pc = pc[out] if out in pc else no_info_pc + pc['device'] = device + result = {out: [infer_dict[out], pc]} + return result + + +@error_handling('getting inference results for outputs: \'{layers}\'') +def overall_accuracy_check(model: str, ref_model: str, out_layers: list, ref_out_layers: list, inputs: dict, + ref_inputs: dict, core: IECore, device: str, ref_core: IECore, ref_device: str, layers: str, + num_of_iterations: int): + global_times, ref_global_times = [], [] + if layers in ['None', None]: + net_copy = get_net_copy_with_output(model=model, output=layers, core=core) + ref_net_copy = get_net_copy_with_output(model=ref_model, output=layers, core=ref_core) + for i in range(num_of_iterations): + t1 = datetime.datetime.now() + infer(net=net_copy, core=core, device=device, inputs=inputs, output=out_layers) + t2 = datetime.datetime.now() + infer(net=ref_net_copy, core=ref_core, device=ref_device, inputs=ref_inputs, output=ref_out_layers) + t3 = datetime.datetime.now() + global_times.append(t2 - t1) + ref_global_times.append(t3 - t2) + return global_times, ref_global_times + + +def one_ir_mode(args): + core = get_plugin(args.device, args.l, args.config) + net = get_net(model=args.model, core=core) + net_layers, net_inputs, net_outputs = get_model_info(net) + log.info(f'{args.device} vs {args.reference_device}') + log.info(f'The same IR on both devices: {args.model}') + out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers) + print_input_layers(net_inputs) + print_output_layers(out_layers) + ref_core = get_plugin(args.reference_device, args.l, args.reference_config) + global_accuracy = [] + inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input) + global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.model, + out_layers=out_layers, ref_out_layers=out_layers, + inputs=inputs, ref_inputs=inputs, core=core, + device=args.device, ref_core=ref_core, + ref_device=args.reference_device, layers=args.layers, + num_of_iterations=args.num_of_iterations) + for out_layer in out_layers: + log.info(f'Layer {out_layer} statistics') + net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core) + results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer]) + if out_layer not in results: + continue + out_blob, pc = results[out_layer] + ref_results = infer(net=net_copy, core=ref_core, device=args.reference_device, + inputs=inputs, output=[out_layer]) + if out_layer not in ref_results: + continue + ref_out_blob, ref_pc = ref_results[out_layer] + a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob) + performance_metrics(pc=pc, ref_pc=ref_pc) + blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob) + global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m) + print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times, + global_accuracy=global_accuracy) + + +def two_ir_mode(args): + core = get_plugin(args.device, args.l, args.config) + ref_core = get_plugin(args.reference_device, args.l, args.reference_config) + net = get_net(model=args.model, core=core) + net_layers, net_inputs, net_outputs = get_model_info(net) + ref_net = get_net(model=args.reference_model, core=ref_core) + ref_net_layers, ref_net_inputs, ref_net_outputs = get_model_info(ref_net) + log.info(f'{args.device} vs {args.reference_device}') + log.info(f'IR for {args.device} : {args.model}') + log.info(f'IR for {args.reference_device} : {args.reference_model}') + out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers) + ref_out_layers = get_layers_list(ref_net_layers, ref_net_inputs, ref_net_outputs, args.layers) + print_input_layers(net_inputs) + print_output_layers(out_layers) + layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping, + user_layers=out_layers) + inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input, + layers_map=layers_map) + ref_inputs = input_processing(model_path=args.reference_model, net_inputs=ref_net_inputs, input_file=args.input, + layers_map=layers_map) + global_accuracy = [] + global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.reference_model, + out_layers=out_layers, ref_out_layers=ref_out_layers, + inputs=inputs, ref_inputs=ref_inputs, core=core, + device=args.device, ref_core=ref_core, + ref_device=args.reference_device, layers=args.layers, + num_of_iterations=args.num_of_iterations) + for out_layer in layers_map: + ref_out_layer = layers_map[out_layer] + if out_layer == ref_out_layer: + log.info(f'Layer {out_layer} statistics') + else: + log.info(f'Statistics \'{out_layer}\' vs \'{ref_out_layer}\'') + net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core) + ref_net_copy = get_net_copy_with_output(model=args.reference_model, output=ref_out_layer, core=ref_core) + results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer]) + if out_layer not in results: + continue + out_blob, pc = results[out_layer] + ref_results = infer(net=ref_net_copy, core=ref_core, device=args.reference_device, + inputs=ref_inputs, output=[ref_out_layer]) + ref_out_blob, ref_pc = ref_results[ref_out_layer] + if ref_out_layer not in ref_results: + continue + a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob) + performance_metrics(pc=pc, ref_pc=ref_pc) + blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob) + global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m) + print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times, + global_accuracy=global_accuracy) + + +def dump_mode(args): + core = get_plugin(args.device, args.l, args.config) + net = get_net(model=args.model, core=core) + func = ng.function_from_cnn(net) + ops = func.get_ops() + out_layers = get_layers_list(ops, net.input_info, net.outputs, args.layers) + inputs = input_processing(args.model, net.input_info, args.input) + dump_dict = {} + for out_layer in out_layers: + log.info(f'Layer {out_layer} processing') + net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core) + results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer]) + if out_layer not in results: + continue + out_blob, pc = results[out_layer] + dump_dict[out_layer] = np.array({'blob': out_blob, 'pc': pc}) + dump_output_file(args.model + '_' + args.device + '_dump.npz', dump_dict) + + +def load_mode(args): + core = get_plugin(args.device, args.l, args.config) + log.info(f'IR for {args.device} : {args.model}') + log.info(f'Loading blob from {args.load}') + net = get_net(model=args.model, core=core) + net_layers, net_inputs, net_outputs = get_model_info(net) + out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers) + print_input_layers(net_inputs) + print_output_layers(out_layers) + layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping, + user_layers=out_layers) + inputs = input_processing(args.model, net_inputs, args.input, layers_map) + global_accuracy = [] + loaded = load_dump(args.load) + for out_layer in layers_map: + ref_out_layer = layers_map[out_layer] + if out_layer == ref_out_layer: + log.info(f'Layer {out_layer} statistics') + else: + log.info(f'Statistics \'{out_layer}\' vs \'{ref_out_layer}\'') + net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core) + results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer]) + if out_layer not in results: + continue + out_blob, pc = results[out_layer] + if ref_out_layer not in loaded: + continue + ref_out_blob = loaded[ref_out_layer]['blob'] + a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob) + if 'pc' in loaded[ref_out_layer]: + ref_pc = loaded[ref_out_layer]['pc'] + performance_metrics(pc=pc, ref_pc=ref_pc) + blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob) + global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m) + print_all_over_the_net_metrics(global_accuracy=global_accuracy) + + +def main(): + set_logger(log.DEBUG) + args = validate_args(build_parser().parse_args()) + + log.info(f'Inference Engine:\n API version ............ {ie.__version__}', extra={'no_lvl': True}) + set_verbosity(args.verbosity) + mode = find_out_cct_mode(args) + if mode == 1: + log.info('Cross check with one IR was enabled') + one_ir_mode(args) + elif mode == 2: + log.info('Cross check with two IRs was enabled') + two_ir_mode(args) + elif mode == 3: + log.info('Dump mode was enabled') + dump_mode(args) + elif mode == 4: + log.info('Load mode was enabled') + load_mode(args) + log.info("Execution successful") + + +if __name__ == '__main__': + main() diff --git a/tools/cross_check_tool/utils.py b/tools/cross_check_tool/openvino/tools/cross_check_tool/utils.py similarity index 100% rename from tools/cross_check_tool/utils.py rename to tools/cross_check_tool/openvino/tools/cross_check_tool/utils.py diff --git a/tools/cross_check_tool/setup.py b/tools/cross_check_tool/setup.py new file mode 100644 index 00000000000..46e8f6dc055 --- /dev/null +++ b/tools/cross_check_tool/setup.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +""" +Use this script to create a wheel with OpenVINO™ Cross Check Tool: + +$ python setup.py sdist bdist_wheel +""" +from pathlib import Path +from setuptools import setup, find_packages + +SETUP_DIR = Path(__file__).resolve().parent + + +def read_text(path): + return (SETUP_DIR / path).read_text() + +setup( + name='cross_check_tool', + version='0.0.0', + author='Intel® Corporation', + license='OSI Approved :: Apache Software License', + author_email='openvino_pushbot@intel.com', + url='https://github.com/openvinotoolkit/openvino', + description='OpenVINO™ Cross Check Tool package', + entry_points={ + 'console_scripts': [ + 'cross_check_tool = openvino.tools.cross_check_tool.cross_check_tool:main'], + }, + classifiers=[ + 'Programming Language :: Python :: 3', + 'OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + ], + packages=find_packages(), + install_requires=read_text('requirements.txt'), + python_requires='>=3.6', +)