[Python Tools] Align namespace for cross_check_tool (#7786)
* align openvino namespace * update README
This commit is contained in:
Julia Kamelina
GitHub
parent
f20f4d024c
commit
7c56155dae
@ -72,6 +72,13 @@ CCT mode arguments:
|
|||||||
--load LOAD Path to a file to load blobs from
|
--load LOAD Path to a file to load blobs from
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Cross Check Tool can also be installed via:
|
||||||
|
```sh
|
||||||
|
$python3 -m pip install <openvino_repo>/tools/cross_check_tool
|
||||||
|
```
|
||||||
|
In this case, to run the tool, call `cross_check_tool` on the command line with necessary parameters.
|
||||||
|
|
||||||
### Examples
|
### Examples
|
||||||
|
|
||||||
1. To check per-layer accuracy and performance of inference in FP32 precision on the CPU against the GPU, run:
|
1. To check per-layer accuracy and performance of inference in FP32 precision on the CPU against the GPU, run:
|
||||||
|
|||||||
@ -3,315 +3,9 @@
|
|||||||
# Copyright (C) 2018-2021 Intel Corporation
|
# Copyright (C) 2018-2021 Intel Corporation
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
import datetime
|
|
||||||
import logging as log
|
|
||||||
import os
|
|
||||||
import sys
|
import sys
|
||||||
|
from openvino.tools.cross_check_tool.cross_check_tool import main
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
try:
|
|
||||||
from openvino import inference_engine as ie
|
|
||||||
from openvino.inference_engine import IENetwork, IECore
|
|
||||||
except Exception as e:
|
|
||||||
exception_type = type(e).__name__
|
|
||||||
print(f"The following error happened while importing Python API module:\n[ {exception_type} ] {e}")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
try:
|
|
||||||
import ngraph as ng
|
|
||||||
except Exception as e:
|
|
||||||
exception_type = type(e).name
|
|
||||||
print(f"The following error happened while importing nGraph module:\n[ {exception_type} ] {e}")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
from utils import get_config_dictionary, get_layers_list, print_output_layers, input_processing, \
|
|
||||||
accuracy_metrics, validate_args, build_parser, set_logger, find_out_cct_mode, print_all_over_the_net_metrics, \
|
|
||||||
update_global_accuracy_matrics, blob_counters, performance_metrics, manage_user_outputs_with_mapping, \
|
|
||||||
dump_output_file, load_dump, error_handling, print_input_layers, set_verbosity
|
|
||||||
|
|
||||||
|
|
||||||
###
|
if __name__ == "__main__":
|
||||||
# PLUGIN
|
sys.exit(main() or 0)
|
||||||
###
|
|
||||||
|
|
||||||
|
|
||||||
@error_handling('plugin of \'{device}\' device config \'{config}\' loading')
|
|
||||||
def set_plugin_config(core: IECore, device: str, config: str = None):
|
|
||||||
core.set_config(get_config_dictionary(config_file=config), device_name=device)
|
|
||||||
|
|
||||||
|
|
||||||
@error_handling('\'{cpu_ext}\' cpu extensions loading')
|
|
||||||
def set_cpu_extensions(core: IECore, cpu_ext: str):
|
|
||||||
core.add_extension(cpu_ext, "CPU")
|
|
||||||
|
|
||||||
|
|
||||||
def get_plugin(device: str, cpu_ext: str = None, config: str = None):
|
|
||||||
ie = IECore()
|
|
||||||
# log.info('{} plugin:\n API version ............ {}'.format(device, plugin.version), extra={'no_lvl': True})
|
|
||||||
set_plugin_config(core=ie, device=device, config=config)
|
|
||||||
if cpu_ext and 'CPU' in device:
|
|
||||||
set_cpu_extensions(core=ie, cpu_ext=cpu_ext)
|
|
||||||
return ie
|
|
||||||
|
|
||||||
|
|
||||||
###
|
|
||||||
# MODEL
|
|
||||||
###
|
|
||||||
|
|
||||||
|
|
||||||
@error_handling('reading {model} IR model')
|
|
||||||
def get_net(model: str, core: IECore):
|
|
||||||
model_xml = model
|
|
||||||
model_bin = os.path.splitext(model_xml)[0] + ".bin"
|
|
||||||
net = core.read_network(model=model_xml, weights=model_bin)
|
|
||||||
return net
|
|
||||||
|
|
||||||
|
|
||||||
@error_handling('loading network to plugin of {device} device')
|
|
||||||
def get_exec_net(core, net, device):
|
|
||||||
return core.load_network(network=net, device_name=device)
|
|
||||||
|
|
||||||
|
|
||||||
@error_handling('output \'{output}\' addition for network from model \'{model}\'')
|
|
||||||
def get_net_copy_with_output(model: str, output: str, core: IECore):
|
|
||||||
net_copy = get_net(model=model, core=core)
|
|
||||||
func = ng.function_from_cnn(net_copy)
|
|
||||||
if output not in ['None', None]:
|
|
||||||
# output with port_id in name is absent in ops list
|
|
||||||
founded_op = [op for op in func.get_ops() if op.friendly_name == output]
|
|
||||||
if founded_op:
|
|
||||||
net_copy.add_outputs(output)
|
|
||||||
else:
|
|
||||||
split = output.rsplit(".", 1)
|
|
||||||
net_copy.add_outputs((split[0], int(split[1])))
|
|
||||||
return net_copy
|
|
||||||
|
|
||||||
|
|
||||||
@error_handling('getting model layers info')
|
|
||||||
def get_model_info(net: IENetwork):
|
|
||||||
func = ng.function_from_cnn(net)
|
|
||||||
ops = func.get_ordered_ops()
|
|
||||||
return ops, net.input_info, net.outputs
|
|
||||||
|
|
||||||
|
|
||||||
###
|
|
||||||
# INFER
|
|
||||||
###
|
|
||||||
|
|
||||||
|
|
||||||
@error_handling('processing inference')
|
|
||||||
def get_infer_results(executable_network, inputs: dict):
|
|
||||||
return executable_network.infer(inputs=inputs)
|
|
||||||
|
|
||||||
|
|
||||||
@error_handling('getting performance counts from executable network')
|
|
||||||
def get_perf_counts(executable_network):
|
|
||||||
return executable_network.requests[0].get_perf_counts()
|
|
||||||
|
|
||||||
|
|
||||||
@error_handling('getting inference results for outputs: \'{output}\' on \'{device}\' device')
|
|
||||||
def infer(net: IENetwork, core: IECore, device: str, inputs: dict, output: list):
|
|
||||||
executable_network = get_exec_net(core=core, net=net, device=device)
|
|
||||||
infer_dict = get_infer_results(executable_network=executable_network, inputs=inputs)
|
|
||||||
pc = get_perf_counts(executable_network=executable_network)
|
|
||||||
no_i = 'no_info'
|
|
||||||
no_info_pc = {'cpu_time': no_i, 'exec_time': no_i, 'layer_type': no_i, 'real_time': no_i, 'status': no_i}
|
|
||||||
result = {}
|
|
||||||
for out in output:
|
|
||||||
if out not in infer_dict:
|
|
||||||
log.warning(f"There is no '{out}' layer in Inference Engine outputs results")
|
|
||||||
continue
|
|
||||||
pc = pc[out] if out in pc else no_info_pc
|
|
||||||
pc['device'] = device
|
|
||||||
result = {out: [infer_dict[out], pc]}
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
@error_handling('getting inference results for outputs: \'{layers}\'')
|
|
||||||
def overall_accuracy_check(model: str, ref_model: str, out_layers: list, ref_out_layers: list, inputs: dict,
|
|
||||||
ref_inputs: dict, core: IECore, device: str, ref_core: IECore, ref_device: str, layers: str,
|
|
||||||
num_of_iterations: int):
|
|
||||||
global_times, ref_global_times = [], []
|
|
||||||
if layers in ['None', None]:
|
|
||||||
net_copy = get_net_copy_with_output(model=model, output=layers, core=core)
|
|
||||||
ref_net_copy = get_net_copy_with_output(model=ref_model, output=layers, core=ref_core)
|
|
||||||
for i in range(num_of_iterations):
|
|
||||||
t1 = datetime.datetime.now()
|
|
||||||
infer(net=net_copy, core=core, device=device, inputs=inputs, output=out_layers)
|
|
||||||
t2 = datetime.datetime.now()
|
|
||||||
infer(net=ref_net_copy, core=ref_core, device=ref_device, inputs=ref_inputs, output=ref_out_layers)
|
|
||||||
t3 = datetime.datetime.now()
|
|
||||||
global_times.append(t2 - t1)
|
|
||||||
ref_global_times.append(t3 - t2)
|
|
||||||
return global_times, ref_global_times
|
|
||||||
|
|
||||||
|
|
||||||
def one_ir_mode(args):
|
|
||||||
core = get_plugin(args.device, args.l, args.config)
|
|
||||||
net = get_net(model=args.model, core=core)
|
|
||||||
net_layers, net_inputs, net_outputs = get_model_info(net)
|
|
||||||
log.info(f'{args.device} vs {args.reference_device}')
|
|
||||||
log.info(f'The same IR on both devices: {args.model}')
|
|
||||||
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
|
|
||||||
print_input_layers(net_inputs)
|
|
||||||
print_output_layers(out_layers)
|
|
||||||
ref_core = get_plugin(args.reference_device, args.l, args.reference_config)
|
|
||||||
global_accuracy = []
|
|
||||||
inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input)
|
|
||||||
global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.model,
|
|
||||||
out_layers=out_layers, ref_out_layers=out_layers,
|
|
||||||
inputs=inputs, ref_inputs=inputs, core=core,
|
|
||||||
device=args.device, ref_core=ref_core,
|
|
||||||
ref_device=args.reference_device, layers=args.layers,
|
|
||||||
num_of_iterations=args.num_of_iterations)
|
|
||||||
for out_layer in out_layers:
|
|
||||||
log.info(f'Layer {out_layer} statistics')
|
|
||||||
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
|
|
||||||
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
|
|
||||||
if out_layer not in results:
|
|
||||||
continue
|
|
||||||
out_blob, pc = results[out_layer]
|
|
||||||
ref_results = infer(net=net_copy, core=ref_core, device=args.reference_device,
|
|
||||||
inputs=inputs, output=[out_layer])
|
|
||||||
if out_layer not in ref_results:
|
|
||||||
continue
|
|
||||||
ref_out_blob, ref_pc = ref_results[out_layer]
|
|
||||||
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
|
|
||||||
performance_metrics(pc=pc, ref_pc=ref_pc)
|
|
||||||
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
|
|
||||||
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
|
|
||||||
print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times,
|
|
||||||
global_accuracy=global_accuracy)
|
|
||||||
|
|
||||||
|
|
||||||
def two_ir_mode(args):
|
|
||||||
core = get_plugin(args.device, args.l, args.config)
|
|
||||||
ref_core = get_plugin(args.reference_device, args.l, args.reference_config)
|
|
||||||
net = get_net(model=args.model, core=core)
|
|
||||||
net_layers, net_inputs, net_outputs = get_model_info(net)
|
|
||||||
ref_net = get_net(model=args.reference_model, core=ref_core)
|
|
||||||
ref_net_layers, ref_net_inputs, ref_net_outputs = get_model_info(ref_net)
|
|
||||||
log.info(f'{args.device} vs {args.reference_device}')
|
|
||||||
log.info(f'IR for {args.device} : {args.model}')
|
|
||||||
log.info(f'IR for {args.reference_device} : {args.reference_model}')
|
|
||||||
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
|
|
||||||
ref_out_layers = get_layers_list(ref_net_layers, ref_net_inputs, ref_net_outputs, args.layers)
|
|
||||||
print_input_layers(net_inputs)
|
|
||||||
print_output_layers(out_layers)
|
|
||||||
layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping,
|
|
||||||
user_layers=out_layers)
|
|
||||||
inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input,
|
|
||||||
layers_map=layers_map)
|
|
||||||
ref_inputs = input_processing(model_path=args.reference_model, net_inputs=ref_net_inputs, input_file=args.input,
|
|
||||||
layers_map=layers_map)
|
|
||||||
global_accuracy = []
|
|
||||||
global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.reference_model,
|
|
||||||
out_layers=out_layers, ref_out_layers=ref_out_layers,
|
|
||||||
inputs=inputs, ref_inputs=ref_inputs, core=core,
|
|
||||||
device=args.device, ref_core=ref_core,
|
|
||||||
ref_device=args.reference_device, layers=args.layers,
|
|
||||||
num_of_iterations=args.num_of_iterations)
|
|
||||||
for out_layer in layers_map:
|
|
||||||
ref_out_layer = layers_map[out_layer]
|
|
||||||
if out_layer == ref_out_layer:
|
|
||||||
log.info(f'Layer {out_layer} statistics')
|
|
||||||
else:
|
|
||||||
log.info(f'Statistics \'{out_layer}\' vs \'{ref_out_layer}\'')
|
|
||||||
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
|
|
||||||
ref_net_copy = get_net_copy_with_output(model=args.reference_model, output=ref_out_layer, core=ref_core)
|
|
||||||
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
|
|
||||||
if out_layer not in results:
|
|
||||||
continue
|
|
||||||
out_blob, pc = results[out_layer]
|
|
||||||
ref_results = infer(net=ref_net_copy, core=ref_core, device=args.reference_device,
|
|
||||||
inputs=ref_inputs, output=[ref_out_layer])
|
|
||||||
ref_out_blob, ref_pc = ref_results[ref_out_layer]
|
|
||||||
if ref_out_layer not in ref_results:
|
|
||||||
continue
|
|
||||||
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
|
|
||||||
performance_metrics(pc=pc, ref_pc=ref_pc)
|
|
||||||
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
|
|
||||||
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
|
|
||||||
print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times,
|
|
||||||
global_accuracy=global_accuracy)
|
|
||||||
|
|
||||||
|
|
||||||
def dump_mode(args):
|
|
||||||
core = get_plugin(args.device, args.l, args.config)
|
|
||||||
net = get_net(model=args.model, core=core)
|
|
||||||
func = ng.function_from_cnn(net)
|
|
||||||
ops = func.get_ops()
|
|
||||||
out_layers = get_layers_list(ops, net.input_info, net.outputs, args.layers)
|
|
||||||
inputs = input_processing(args.model, net.input_info, args.input)
|
|
||||||
dump_dict = {}
|
|
||||||
for out_layer in out_layers:
|
|
||||||
log.info(f'Layer {out_layer} processing')
|
|
||||||
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
|
|
||||||
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
|
|
||||||
if out_layer not in results:
|
|
||||||
continue
|
|
||||||
out_blob, pc = results[out_layer]
|
|
||||||
dump_dict[out_layer] = np.array({'blob': out_blob, 'pc': pc})
|
|
||||||
dump_output_file(args.model + '_' + args.device + '_dump.npz', dump_dict)
|
|
||||||
|
|
||||||
|
|
||||||
def load_mode(args):
|
|
||||||
core = get_plugin(args.device, args.l, args.config)
|
|
||||||
log.info(f'IR for {args.device} : {args.model}')
|
|
||||||
log.info(f'Loading blob from {args.load}')
|
|
||||||
net = get_net(model=args.model, core=core)
|
|
||||||
net_layers, net_inputs, net_outputs = get_model_info(net)
|
|
||||||
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
|
|
||||||
print_input_layers(net_inputs)
|
|
||||||
print_output_layers(out_layers)
|
|
||||||
layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping,
|
|
||||||
user_layers=out_layers)
|
|
||||||
inputs = input_processing(args.model, net_inputs, args.input, layers_map)
|
|
||||||
global_accuracy = []
|
|
||||||
loaded = load_dump(args.load)
|
|
||||||
for out_layer in layers_map:
|
|
||||||
ref_out_layer = layers_map[out_layer]
|
|
||||||
if out_layer == ref_out_layer:
|
|
||||||
log.info(f'Layer {out_layer} statistics')
|
|
||||||
else:
|
|
||||||
log.info(f'Statistics \'{out_layer}\' vs \'{ref_out_layer}\'')
|
|
||||||
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
|
|
||||||
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
|
|
||||||
if out_layer not in results:
|
|
||||||
continue
|
|
||||||
out_blob, pc = results[out_layer]
|
|
||||||
if ref_out_layer not in loaded:
|
|
||||||
continue
|
|
||||||
ref_out_blob = loaded[ref_out_layer]['blob']
|
|
||||||
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
|
|
||||||
if 'pc' in loaded[ref_out_layer]:
|
|
||||||
ref_pc = loaded[ref_out_layer]['pc']
|
|
||||||
performance_metrics(pc=pc, ref_pc=ref_pc)
|
|
||||||
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
|
|
||||||
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
|
|
||||||
print_all_over_the_net_metrics(global_accuracy=global_accuracy)
|
|
||||||
|
|
||||||
|
|
||||||
def main(args):
|
|
||||||
log.info(f'Inference Engine:\n API version ............ {ie.__version__}', extra={'no_lvl': True})
|
|
||||||
set_verbosity(args.verbosity)
|
|
||||||
mode = find_out_cct_mode(args)
|
|
||||||
if mode == 1:
|
|
||||||
log.info('Cross check with one IR was enabled')
|
|
||||||
one_ir_mode(args)
|
|
||||||
elif mode == 2:
|
|
||||||
log.info('Cross check with two IRs was enabled')
|
|
||||||
two_ir_mode(args)
|
|
||||||
elif mode == 3:
|
|
||||||
log.info('Dump mode was enabled')
|
|
||||||
dump_mode(args)
|
|
||||||
elif mode == 4:
|
|
||||||
log.info('Load mode was enabled')
|
|
||||||
load_mode(args)
|
|
||||||
log.info("Execution successful")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
set_logger(log.DEBUG)
|
|
||||||
main(validate_args(build_parser().parse_args()))
|
|
||||||
|
|||||||
3
tools/cross_check_tool/openvino/__init__.py
Normal file
3
tools/cross_check_tool/openvino/__init__.py
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
# Copyright (C) 2018-2021 Intel Corporation
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
__path__ = __import__('pkgutil').extend_path(__path__, __name__)
|
||||||
@ -0,0 +1,3 @@
|
|||||||
|
# Copyright (C) 2018-2021 Intel Corporation
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
319
tools/cross_check_tool/openvino/tools/cross_check_tool/cross_check_tool.py
Executable file
319
tools/cross_check_tool/openvino/tools/cross_check_tool/cross_check_tool.py
Executable file
@ -0,0 +1,319 @@
|
|||||||
|
#!/usr/bin/python3
|
||||||
|
|
||||||
|
# Copyright (C) 2018-2021 Intel Corporation
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
import logging as log
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
try:
|
||||||
|
from openvino import inference_engine as ie
|
||||||
|
from openvino.inference_engine import IENetwork, IECore
|
||||||
|
except Exception as e:
|
||||||
|
exception_type = type(e).__name__
|
||||||
|
print(f"The following error happened while importing Python API module:\n[ {exception_type} ] {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
import ngraph as ng
|
||||||
|
except Exception as e:
|
||||||
|
exception_type = type(e).name
|
||||||
|
print(f"The following error happened while importing nGraph module:\n[ {exception_type} ] {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
from openvino.tools.cross_check_tool.utils import get_config_dictionary, get_layers_list, print_output_layers, \
|
||||||
|
input_processing, accuracy_metrics, validate_args, build_parser, set_logger, find_out_cct_mode, \
|
||||||
|
print_all_over_the_net_metrics, update_global_accuracy_matrics, blob_counters, performance_metrics, \
|
||||||
|
manage_user_outputs_with_mapping, dump_output_file, load_dump, error_handling, print_input_layers, set_verbosity
|
||||||
|
|
||||||
|
|
||||||
|
###
|
||||||
|
# PLUGIN
|
||||||
|
###
|
||||||
|
|
||||||
|
|
||||||
|
@error_handling('plugin of \'{device}\' device config \'{config}\' loading')
|
||||||
|
def set_plugin_config(core: IECore, device: str, config: str = None):
|
||||||
|
core.set_config(get_config_dictionary(config_file=config), device_name=device)
|
||||||
|
|
||||||
|
|
||||||
|
@error_handling('\'{cpu_ext}\' cpu extensions loading')
|
||||||
|
def set_cpu_extensions(core: IECore, cpu_ext: str):
|
||||||
|
core.add_extension(cpu_ext, "CPU")
|
||||||
|
|
||||||
|
|
||||||
|
def get_plugin(device: str, cpu_ext: str = None, config: str = None):
|
||||||
|
ie = IECore()
|
||||||
|
# log.info('{} plugin:\n API version ............ {}'.format(device, plugin.version), extra={'no_lvl': True})
|
||||||
|
set_plugin_config(core=ie, device=device, config=config)
|
||||||
|
if cpu_ext and 'CPU' in device:
|
||||||
|
set_cpu_extensions(core=ie, cpu_ext=cpu_ext)
|
||||||
|
return ie
|
||||||
|
|
||||||
|
|
||||||
|
###
|
||||||
|
# MODEL
|
||||||
|
###
|
||||||
|
|
||||||
|
|
||||||
|
@error_handling('reading {model} IR model')
|
||||||
|
def get_net(model: str, core: IECore):
|
||||||
|
model_xml = model
|
||||||
|
model_bin = os.path.splitext(model_xml)[0] + ".bin"
|
||||||
|
net = core.read_network(model=model_xml, weights=model_bin)
|
||||||
|
return net
|
||||||
|
|
||||||
|
|
||||||
|
@error_handling('loading network to plugin of {device} device')
|
||||||
|
def get_exec_net(core, net, device):
|
||||||
|
return core.load_network(network=net, device_name=device)
|
||||||
|
|
||||||
|
|
||||||
|
@error_handling('output \'{output}\' addition for network from model \'{model}\'')
|
||||||
|
def get_net_copy_with_output(model: str, output: str, core: IECore):
|
||||||
|
net_copy = get_net(model=model, core=core)
|
||||||
|
func = ng.function_from_cnn(net_copy)
|
||||||
|
if output not in ['None', None]:
|
||||||
|
# output with port_id in name is absent in ops list
|
||||||
|
founded_op = [op for op in func.get_ops() if op.friendly_name == output]
|
||||||
|
if founded_op:
|
||||||
|
net_copy.add_outputs(output)
|
||||||
|
else:
|
||||||
|
split = output.rsplit(".", 1)
|
||||||
|
net_copy.add_outputs((split[0], int(split[1])))
|
||||||
|
return net_copy
|
||||||
|
|
||||||
|
|
||||||
|
@error_handling('getting model layers info')
|
||||||
|
def get_model_info(net: IENetwork):
|
||||||
|
func = ng.function_from_cnn(net)
|
||||||
|
ops = func.get_ordered_ops()
|
||||||
|
return ops, net.input_info, net.outputs
|
||||||
|
|
||||||
|
|
||||||
|
###
|
||||||
|
# INFER
|
||||||
|
###
|
||||||
|
|
||||||
|
|
||||||
|
@error_handling('processing inference')
|
||||||
|
def get_infer_results(executable_network, inputs: dict):
|
||||||
|
return executable_network.infer(inputs=inputs)
|
||||||
|
|
||||||
|
|
||||||
|
@error_handling('getting performance counts from executable network')
|
||||||
|
def get_perf_counts(executable_network):
|
||||||
|
return executable_network.requests[0].get_perf_counts()
|
||||||
|
|
||||||
|
|
||||||
|
@error_handling('getting inference results for outputs: \'{output}\' on \'{device}\' device')
|
||||||
|
def infer(net: IENetwork, core: IECore, device: str, inputs: dict, output: list):
|
||||||
|
executable_network = get_exec_net(core=core, net=net, device=device)
|
||||||
|
infer_dict = get_infer_results(executable_network=executable_network, inputs=inputs)
|
||||||
|
pc = get_perf_counts(executable_network=executable_network)
|
||||||
|
no_i = 'no_info'
|
||||||
|
no_info_pc = {'cpu_time': no_i, 'exec_time': no_i, 'layer_type': no_i, 'real_time': no_i, 'status': no_i}
|
||||||
|
result = {}
|
||||||
|
for out in output:
|
||||||
|
if out not in infer_dict:
|
||||||
|
log.warning(f"There is no '{out}' layer in Inference Engine outputs results")
|
||||||
|
continue
|
||||||
|
pc = pc[out] if out in pc else no_info_pc
|
||||||
|
pc['device'] = device
|
||||||
|
result = {out: [infer_dict[out], pc]}
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@error_handling('getting inference results for outputs: \'{layers}\'')
|
||||||
|
def overall_accuracy_check(model: str, ref_model: str, out_layers: list, ref_out_layers: list, inputs: dict,
|
||||||
|
ref_inputs: dict, core: IECore, device: str, ref_core: IECore, ref_device: str, layers: str,
|
||||||
|
num_of_iterations: int):
|
||||||
|
global_times, ref_global_times = [], []
|
||||||
|
if layers in ['None', None]:
|
||||||
|
net_copy = get_net_copy_with_output(model=model, output=layers, core=core)
|
||||||
|
ref_net_copy = get_net_copy_with_output(model=ref_model, output=layers, core=ref_core)
|
||||||
|
for i in range(num_of_iterations):
|
||||||
|
t1 = datetime.datetime.now()
|
||||||
|
infer(net=net_copy, core=core, device=device, inputs=inputs, output=out_layers)
|
||||||
|
t2 = datetime.datetime.now()
|
||||||
|
infer(net=ref_net_copy, core=ref_core, device=ref_device, inputs=ref_inputs, output=ref_out_layers)
|
||||||
|
t3 = datetime.datetime.now()
|
||||||
|
global_times.append(t2 - t1)
|
||||||
|
ref_global_times.append(t3 - t2)
|
||||||
|
return global_times, ref_global_times
|
||||||
|
|
||||||
|
|
||||||
|
def one_ir_mode(args):
|
||||||
|
core = get_plugin(args.device, args.l, args.config)
|
||||||
|
net = get_net(model=args.model, core=core)
|
||||||
|
net_layers, net_inputs, net_outputs = get_model_info(net)
|
||||||
|
log.info(f'{args.device} vs {args.reference_device}')
|
||||||
|
log.info(f'The same IR on both devices: {args.model}')
|
||||||
|
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
|
||||||
|
print_input_layers(net_inputs)
|
||||||
|
print_output_layers(out_layers)
|
||||||
|
ref_core = get_plugin(args.reference_device, args.l, args.reference_config)
|
||||||
|
global_accuracy = []
|
||||||
|
inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input)
|
||||||
|
global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.model,
|
||||||
|
out_layers=out_layers, ref_out_layers=out_layers,
|
||||||
|
inputs=inputs, ref_inputs=inputs, core=core,
|
||||||
|
device=args.device, ref_core=ref_core,
|
||||||
|
ref_device=args.reference_device, layers=args.layers,
|
||||||
|
num_of_iterations=args.num_of_iterations)
|
||||||
|
for out_layer in out_layers:
|
||||||
|
log.info(f'Layer {out_layer} statistics')
|
||||||
|
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
|
||||||
|
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
|
||||||
|
if out_layer not in results:
|
||||||
|
continue
|
||||||
|
out_blob, pc = results[out_layer]
|
||||||
|
ref_results = infer(net=net_copy, core=ref_core, device=args.reference_device,
|
||||||
|
inputs=inputs, output=[out_layer])
|
||||||
|
if out_layer not in ref_results:
|
||||||
|
continue
|
||||||
|
ref_out_blob, ref_pc = ref_results[out_layer]
|
||||||
|
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
|
||||||
|
performance_metrics(pc=pc, ref_pc=ref_pc)
|
||||||
|
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
|
||||||
|
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
|
||||||
|
print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times,
|
||||||
|
global_accuracy=global_accuracy)
|
||||||
|
|
||||||
|
|
||||||
|
def two_ir_mode(args):
|
||||||
|
core = get_plugin(args.device, args.l, args.config)
|
||||||
|
ref_core = get_plugin(args.reference_device, args.l, args.reference_config)
|
||||||
|
net = get_net(model=args.model, core=core)
|
||||||
|
net_layers, net_inputs, net_outputs = get_model_info(net)
|
||||||
|
ref_net = get_net(model=args.reference_model, core=ref_core)
|
||||||
|
ref_net_layers, ref_net_inputs, ref_net_outputs = get_model_info(ref_net)
|
||||||
|
log.info(f'{args.device} vs {args.reference_device}')
|
||||||
|
log.info(f'IR for {args.device} : {args.model}')
|
||||||
|
log.info(f'IR for {args.reference_device} : {args.reference_model}')
|
||||||
|
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
|
||||||
|
ref_out_layers = get_layers_list(ref_net_layers, ref_net_inputs, ref_net_outputs, args.layers)
|
||||||
|
print_input_layers(net_inputs)
|
||||||
|
print_output_layers(out_layers)
|
||||||
|
layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping,
|
||||||
|
user_layers=out_layers)
|
||||||
|
inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input,
|
||||||
|
layers_map=layers_map)
|
||||||
|
ref_inputs = input_processing(model_path=args.reference_model, net_inputs=ref_net_inputs, input_file=args.input,
|
||||||
|
layers_map=layers_map)
|
||||||
|
global_accuracy = []
|
||||||
|
global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.reference_model,
|
||||||
|
out_layers=out_layers, ref_out_layers=ref_out_layers,
|
||||||
|
inputs=inputs, ref_inputs=ref_inputs, core=core,
|
||||||
|
device=args.device, ref_core=ref_core,
|
||||||
|
ref_device=args.reference_device, layers=args.layers,
|
||||||
|
num_of_iterations=args.num_of_iterations)
|
||||||
|
for out_layer in layers_map:
|
||||||
|
ref_out_layer = layers_map[out_layer]
|
||||||
|
if out_layer == ref_out_layer:
|
||||||
|
log.info(f'Layer {out_layer} statistics')
|
||||||
|
else:
|
||||||
|
log.info(f'Statistics \'{out_layer}\' vs \'{ref_out_layer}\'')
|
||||||
|
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
|
||||||
|
ref_net_copy = get_net_copy_with_output(model=args.reference_model, output=ref_out_layer, core=ref_core)
|
||||||
|
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
|
||||||
|
if out_layer not in results:
|
||||||
|
continue
|
||||||
|
out_blob, pc = results[out_layer]
|
||||||
|
ref_results = infer(net=ref_net_copy, core=ref_core, device=args.reference_device,
|
||||||
|
inputs=ref_inputs, output=[ref_out_layer])
|
||||||
|
ref_out_blob, ref_pc = ref_results[ref_out_layer]
|
||||||
|
if ref_out_layer not in ref_results:
|
||||||
|
continue
|
||||||
|
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
|
||||||
|
performance_metrics(pc=pc, ref_pc=ref_pc)
|
||||||
|
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
|
||||||
|
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
|
||||||
|
print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times,
|
||||||
|
global_accuracy=global_accuracy)
|
||||||
|
|
||||||
|
|
||||||
|
def dump_mode(args):
|
||||||
|
core = get_plugin(args.device, args.l, args.config)
|
||||||
|
net = get_net(model=args.model, core=core)
|
||||||
|
func = ng.function_from_cnn(net)
|
||||||
|
ops = func.get_ops()
|
||||||
|
out_layers = get_layers_list(ops, net.input_info, net.outputs, args.layers)
|
||||||
|
inputs = input_processing(args.model, net.input_info, args.input)
|
||||||
|
dump_dict = {}
|
||||||
|
for out_layer in out_layers:
|
||||||
|
log.info(f'Layer {out_layer} processing')
|
||||||
|
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
|
||||||
|
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
|
||||||
|
if out_layer not in results:
|
||||||
|
continue
|
||||||
|
out_blob, pc = results[out_layer]
|
||||||
|
dump_dict[out_layer] = np.array({'blob': out_blob, 'pc': pc})
|
||||||
|
dump_output_file(args.model + '_' + args.device + '_dump.npz', dump_dict)
|
||||||
|
|
||||||
|
|
||||||
|
def load_mode(args):
|
||||||
|
core = get_plugin(args.device, args.l, args.config)
|
||||||
|
log.info(f'IR for {args.device} : {args.model}')
|
||||||
|
log.info(f'Loading blob from {args.load}')
|
||||||
|
net = get_net(model=args.model, core=core)
|
||||||
|
net_layers, net_inputs, net_outputs = get_model_info(net)
|
||||||
|
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
|
||||||
|
print_input_layers(net_inputs)
|
||||||
|
print_output_layers(out_layers)
|
||||||
|
layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping,
|
||||||
|
user_layers=out_layers)
|
||||||
|
inputs = input_processing(args.model, net_inputs, args.input, layers_map)
|
||||||
|
global_accuracy = []
|
||||||
|
loaded = load_dump(args.load)
|
||||||
|
for out_layer in layers_map:
|
||||||
|
ref_out_layer = layers_map[out_layer]
|
||||||
|
if out_layer == ref_out_layer:
|
||||||
|
log.info(f'Layer {out_layer} statistics')
|
||||||
|
else:
|
||||||
|
log.info(f'Statistics \'{out_layer}\' vs \'{ref_out_layer}\'')
|
||||||
|
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
|
||||||
|
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
|
||||||
|
if out_layer not in results:
|
||||||
|
continue
|
||||||
|
out_blob, pc = results[out_layer]
|
||||||
|
if ref_out_layer not in loaded:
|
||||||
|
continue
|
||||||
|
ref_out_blob = loaded[ref_out_layer]['blob']
|
||||||
|
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
|
||||||
|
if 'pc' in loaded[ref_out_layer]:
|
||||||
|
ref_pc = loaded[ref_out_layer]['pc']
|
||||||
|
performance_metrics(pc=pc, ref_pc=ref_pc)
|
||||||
|
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
|
||||||
|
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
|
||||||
|
print_all_over_the_net_metrics(global_accuracy=global_accuracy)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
set_logger(log.DEBUG)
|
||||||
|
args = validate_args(build_parser().parse_args())
|
||||||
|
|
||||||
|
log.info(f'Inference Engine:\n API version ............ {ie.__version__}', extra={'no_lvl': True})
|
||||||
|
set_verbosity(args.verbosity)
|
||||||
|
mode = find_out_cct_mode(args)
|
||||||
|
if mode == 1:
|
||||||
|
log.info('Cross check with one IR was enabled')
|
||||||
|
one_ir_mode(args)
|
||||||
|
elif mode == 2:
|
||||||
|
log.info('Cross check with two IRs was enabled')
|
||||||
|
two_ir_mode(args)
|
||||||
|
elif mode == 3:
|
||||||
|
log.info('Dump mode was enabled')
|
||||||
|
dump_mode(args)
|
||||||
|
elif mode == 4:
|
||||||
|
log.info('Load mode was enabled')
|
||||||
|
load_mode(args)
|
||||||
|
log.info("Execution successful")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
40
tools/cross_check_tool/setup.py
Normal file
40
tools/cross_check_tool/setup.py
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
# Copyright (C) 2018-2021 Intel Corporation
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
"""
|
||||||
|
Use this script to create a wheel with OpenVINO™ Cross Check Tool:
|
||||||
|
|
||||||
|
$ python setup.py sdist bdist_wheel
|
||||||
|
"""
|
||||||
|
from pathlib import Path
|
||||||
|
from setuptools import setup, find_packages
|
||||||
|
|
||||||
|
SETUP_DIR = Path(__file__).resolve().parent
|
||||||
|
|
||||||
|
|
||||||
|
def read_text(path):
|
||||||
|
return (SETUP_DIR / path).read_text()
|
||||||
|
|
||||||
|
setup(
|
||||||
|
name='cross_check_tool',
|
||||||
|
version='0.0.0',
|
||||||
|
author='Intel® Corporation',
|
||||||
|
license='OSI Approved :: Apache Software License',
|
||||||
|
author_email='openvino_pushbot@intel.com',
|
||||||
|
url='https://github.com/openvinotoolkit/openvino',
|
||||||
|
description='OpenVINO™ Cross Check Tool package',
|
||||||
|
entry_points={
|
||||||
|
'console_scripts': [
|
||||||
|
'cross_check_tool = openvino.tools.cross_check_tool.cross_check_tool:main'],
|
||||||
|
},
|
||||||
|
classifiers=[
|
||||||
|
'Programming Language :: Python :: 3',
|
||||||
|
'OSI Approved :: Apache Software License',
|
||||||
|
'Operating System :: OS Independent',
|
||||||
|
],
|
||||||
|
packages=find_packages(),
|
||||||
|
install_requires=read_text('requirements.txt'),
|
||||||
|
python_requires='>=3.6',
|
||||||
|
)
|
||||||
Loading…
Reference in New Issue
Block a user