IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[tools] cross check tool with api 2.0 (#10058)

Browse Source 
* save work

* save work

* save work

* basic changes with api 2.0

* Support input file mapping and bin files

* Some impovements

* remove mapping support

* Add -ref_layers parameter

* Fix error handler

* Update Readme and remove old parameters

* Fix readme

* remove info about precision

* rename layer to op

* rename blob to tensor

* remove info about shape

* remove unused imports
This commit is contained in:
Alexey Lebedev
2022-02-14 20:25:31 +03:00
committed by GitHub
parent 0050643e9b
commit d3712a148b
4 changed files with 365 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tools/cross_check_tool/README.md
View File

@@ -26,8 +26,6 @@ Model specific arguments:
Path to an input image file or multi-input file to
infer. Generates input(s) from normal distribution if
empty
--batch BATCH, -b BATCH
Overrides batch size. Default is inherited from model
--model MODEL, -m MODEL
Path to an .xml file that represents the first IR of
the trained model to infer.
@@ -38,11 +36,11 @@ Model specific arguments:
Defines layers to check. Options: all, None - for
output layers check, list of comma-separated layer
names to check. Default value is None.
--mapping MAPPING, -map MAPPING
Model Optimizer provided mapping for --model/-m
--reference_mapping REFERENCE_MAPPING, -ref_map REFERENCE_MAPPING
Model Optimizer provided mapping for
--reference_model/-ref_model
--ref_layers REFERENCE_LAYERS, -reference_layers REFERENCE_LAYERS
Defines layers to check in reference model. Options: all, None - for
output layers check, list of comma-separated layer
names to check. If not specified the same layers will
be processed as in --layers parameter.
--num_of_iterations NUM_OF_ITERATIONS, -ni NUM_OF_ITERATIONS
Number of iterations to collect all over the net
performance

360
tools/cross_check_tool/openvino/tools/cross_check_tool/cross_check_tool.py
View File

@@ -5,30 +5,22 @@
import datetime
import logging as log
import os
import sys
from collections import defaultdict
from typing import Union
import numpy as np
try:
from openvino import inference_engine as ie
from openvino.inference_engine import IENetwork, IECore
from openvino.runtime import Core, Model, CompiledModel, InferRequest, Output, get_version
except Exception as e:
exception_type = type(e).__name__
print(f"The following error happened while importing Python API module:\n[ {exception_type} ] {e}")
print(f"The following error happened while importing OpenVINO Python API module:\n[ {exception_type} ] {e}")
sys.exit(1)
try:
import ngraph as ng
except Exception as e:
exception_type = type(e).name
print(f"The following error happened while importing nGraph module:\n[ {exception_type} ] {e}")
sys.exit(1)
from openvino.tools.cross_check_tool.utils import get_config_dictionary, get_layers_list, print_output_layers, \
from openvino.tools.cross_check_tool.utils import get_config_dictionary, get_ops_list, print_output_ops, \
input_processing, accuracy_metrics, validate_args, build_parser, set_logger, find_out_cct_mode, \
print_all_over_the_net_metrics, update_global_accuracy_matrics, blob_counters, performance_metrics, \
manage_user_outputs_with_mapping, dump_output_file, load_dump, error_handling, print_input_layers, set_verbosity
print_all_over_the_net_metrics, update_global_accuracy_matrics, tensor_counters, performance_metrics, \
dump_output_file, load_dump, error_handling, print_inputs, set_verbosity, perf_counts_to_dump, load_profiling_info
###
@@ -37,22 +29,22 @@ from openvino.tools.cross_check_tool.utils import get_config_dictionary, get_lay
@error_handling('plugin of \'{device}\' device config \'{config}\' loading')
def set_plugin_config(core: IECore, device: str, config: str = None):
core.set_config(get_config_dictionary(config_file=config), device_name=device)
def set_plugin_config(core: Core, device: str, config: str = None):
core.set_property(device, get_config_dictionary(config_file=config))
@error_handling('\'{cpu_ext}\' cpu extensions loading')
def set_cpu_extensions(core: IECore, cpu_ext: str):
core.add_extension(cpu_ext, "CPU")
def set_cpu_extensions(core: Core, cpu_ext: str):
core.add_extension(cpu_ext)
def get_plugin(device: str, cpu_ext: str = None, config: str = None):
ie = IECore()
core = Core()
# log.info('{} plugin:\n API version ............ {}'.format(device, plugin.version), extra={'no_lvl': True})
set_plugin_config(core=ie, device=device, config=config)
set_plugin_config(core=core, device=device, config=config)
if cpu_ext and 'CPU' in device:
set_cpu_extensions(core=ie, cpu_ext=cpu_ext)
return ie
set_cpu_extensions(core=core, cpu_ext=cpu_ext)
return core
###
@@ -60,87 +52,108 @@ def get_plugin(device: str, cpu_ext: str = None, config: str = None):
###
@error_handling('reading {model} IR model')
def get_net(model: str, core: IECore):
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
net = core.read_network(model=model_xml, weights=model_bin)
return net
@error_handling('reading {xml_path} IR model')
def get_model(model_path: str, core: Core):
model = core.read_model(model=model_path)
# TODO: can we support it?
if model.is_dynamic():
raise Exception("Cross check tool doesn't support dynamic models for now.")
return model
@error_handling('loading network to plugin of {device} device')
def get_exec_net(core, net, device):
return core.load_network(network=net, device_name=device)
@error_handling('compiling model for {device} device')
def get_compiled_model(core: Core, model: Model, device: str):
return core.compile_model(model=model, device_name=device)
@error_handling('creating infer request')
def get_infer_request(compiled_model: CompiledModel):
return compiled_model.create_infer_request()
@error_handling('output \'{output}\' addition for network from model \'{model}\'')
def get_net_copy_with_output(model: str, output: str, core: IECore):
net_copy = get_net(model=model, core=core)
func = ng.function_from_cnn(net_copy)
def get_model_copy_with_output(model: str, output: tuple, core: Core):
model_copy = get_model(model_path=model, core=core)
new_output = None
if output not in ['None', None]:
# output with port_id in name is absent in ops list
founded_op = [op for op in func.get_ops() if op.friendly_name == output]
if founded_op:
net_copy.add_outputs(output)
else:
split = output.rsplit(".", 1)
net_copy.add_outputs((split[0], int(split[1])))
return net_copy
new_output = model_copy.add_outputs(output).pop()
return model_copy, new_output
@error_handling('getting model layers info')
def get_model_info(net: IENetwork):
func = ng.function_from_cnn(net)
ops = func.get_ordered_ops()
return ops, net.input_info, net.outputs
@error_handling('getting model operations info')
def get_model_info(model: Model):
return model.get_ordered_ops(), model.inputs, model.outputs
def check_inputs_and_default_outputs_are_equal(model, ref_model):
if len(model.inputs) != len(ref_model.inputs):
raise Exception("Models have different number of inputs! Cannot cross check!")
if len(model.outputs) != len(ref_model.outputs):
raise Exception("Models have different number of outputs! Cannot cross check!")
for input, ref_input in zip(model.inputs, ref_model.inputs):
if input.any_name != ref_input.any_name:
raise Exception("Models have different inputs! Cannot cross check!")
for output, ref_output in zip(model.outputs, ref_model.outputs):
if output.any_name != ref_output.any_name:
raise Exception("Models have different outputs! Cannot cross check!")
def get_ops_intersection(ops, ref_ops):
ops_map = {node.friendly_name: node for node in ops}
operation_names = set(ops_map.keys())
ref_operation_names = set(node.friendly_name for node in ref_ops)
intersection_names = operation_names.intersection(ref_operation_names)
return [ops_map[intersection_name] for intersection_name in intersection_names]
def get_ops_union(ops, ref_ops):
ops_map = {}
for op, ref_op in zip(ops, ref_ops):
ops_map.update({op.friendly_name: op})
ops_map.update({ref_op.friendly_name: ref_op})
return ops_map.values()
###
# INFER
###
@error_handling('processing inference')
def get_infer_results(executable_network, inputs: dict):
return executable_network.infer(inputs=inputs)
@error_handling('getting inference results for output: \'{output.any_name}\'')
def get_infer_results(infer_request: InferRequest, output: Output):
return infer_request.get_tensor(output).data
@error_handling('getting performance counts from executable network')
def get_perf_counts(executable_network):
return executable_network.requests[0].get_perf_counts()
@error_handling('getting performance counts from infer request')
def get_profiling_info(infer_request: InferRequest, port: Output):
for pi in infer_request.profiling_info:
if pi.node_name == port.node.friendly_name:
return pi
@error_handling('getting inference results for outputs: \'{output}\' on \'{device}\' device')
def infer(net: IENetwork, core: IECore, device: str, inputs: dict, output: list):
executable_network = get_exec_net(core=core, net=net, device=device)
infer_dict = get_infer_results(executable_network=executable_network, inputs=inputs)
pc = get_perf_counts(executable_network=executable_network)
no_i = 'no_info'
no_info_pc = {'cpu_time': no_i, 'exec_time': no_i, 'layer_type': no_i, 'real_time': no_i, 'status': no_i}
result = {}
for out in output:
if out not in infer_dict:
log.warning(f"There is no '{out}' layer in Inference Engine outputs results")
continue
pc = pc[out] if out in pc else no_info_pc
pc['device'] = device
result = {out: [infer_dict[out], pc]}
return result
@error_handling('processing inference on \'{device}\' device')
def infer(model: Model, core: Core, device: str, inputs: Union[list, dict], output=None):
compiled_model = get_compiled_model(core=core, model=model, device=device)
infer_request = get_infer_request(compiled_model)
infer_request.infer(inputs)
if output:
result = get_infer_results(infer_request, output)
prof_info = get_profiling_info(infer_request, output)
return result, prof_info
@error_handling('getting inference results for outputs: \'{layers}\'')
def overall_accuracy_check(model: str, ref_model: str, out_layers: list, ref_out_layers: list, inputs: dict,
ref_inputs: dict, core: IECore, device: str, ref_core: IECore, ref_device: str, layers: str,
@error_handling('computing overall performance')
def overall_accuracy_check(model: str, ref_model: str, out_ops: list, ref_out_ops: list, inputs: list,
ref_inputs: list, core: Core, device: str, ref_core: Core, ref_device: str, layers: str,
num_of_iterations: int):
global_times, ref_global_times = [], []
if layers in ['None', None]:
net_copy = get_net_copy_with_output(model=model, output=layers, core=core)
ref_net_copy = get_net_copy_with_output(model=ref_model, output=layers, core=ref_core)
model_copy, _ = get_model_copy_with_output(model=model, output=layers, core=core)
ref_model_copy, _ = get_model_copy_with_output(model=ref_model, output=layers, core=ref_core)
for i in range(num_of_iterations):
t1 = datetime.datetime.now()
infer(net=net_copy, core=core, device=device, inputs=inputs, output=out_layers)
infer(model=model_copy, core=core, device=device, inputs=inputs)
t2 = datetime.datetime.now()
infer(net=ref_net_copy, core=ref_core, device=ref_device, inputs=ref_inputs, output=ref_out_layers)
infer(model=ref_model_copy, core=ref_core, device=ref_device, inputs=ref_inputs)
t3 = datetime.datetime.now()
global_times.append(t2 - t1)
ref_global_times.append(t3 - t2)
@@ -149,38 +162,34 @@ def overall_accuracy_check(model: str, ref_model: str, out_layers: list, ref_out
def one_ir_mode(args):
core = get_plugin(args.device, args.l, args.config)
net = get_net(model=args.model, core=core)
net_layers, net_inputs, net_outputs = get_model_info(net)
model = get_model(model_path=args.model, core=core)
model_ops, model_inputs, model_outputs = get_model_info(model)
log.info(f'{args.device} vs {args.reference_device}')
log.info(f'The same IR on both devices: {args.model}')
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
print_input_layers(net_inputs)
print_output_layers(out_layers)
out_ops = get_ops_list(model_ops, model_outputs, args.layers)
print_inputs(model_inputs)
print_output_ops(out_ops)
ref_core = get_plugin(args.reference_device, args.l, args.reference_config)
global_accuracy = []
inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input)
inputs = input_processing(model_path=args.model, model_inputs=model_inputs, input_file=args.input)
global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.model,
out_layers=out_layers, ref_out_layers=out_layers,
out_ops=out_ops, ref_out_ops=out_ops,
inputs=inputs, ref_inputs=inputs, core=core,
device=args.device, ref_core=ref_core,
ref_device=args.reference_device, layers=args.layers,
num_of_iterations=args.num_of_iterations)
for out_layer in out_layers:
log.info(f'Layer {out_layer} statistics')
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
if out_layer not in results:
continue
out_blob, pc = results[out_layer]
ref_results = infer(net=net_copy, core=ref_core, device=args.reference_device,
inputs=inputs, output=[out_layer])
if out_layer not in ref_results:
continue
ref_out_blob, ref_pc = ref_results[out_layer]
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
performance_metrics(pc=pc, ref_pc=ref_pc)
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
for op in out_ops:
log.info(f'Layer {op.friendly_name} statistics')
for i in range(op.get_output_size()):
if op.get_output_size() > 1:
log.info(f'Port {i}: ')
model_copy, new_output = get_model_copy_with_output(model=args.model, output=(op.friendly_name, i), core=core)
out_tensor, pc = infer(model=model_copy, core=core, device=args.device, inputs=inputs, output=new_output)
ref_out_tensor, ref_pc = infer(model=model_copy, core=ref_core, device=args.reference_device, inputs=inputs, output=new_output)
a_m = accuracy_metrics(out_tensor, ref_out_tensor)
performance_metrics(args.device, pc, args.reference_device, ref_pc)
tensor_counters(out_tensor, ref_out_tensor)
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times,
global_accuracy=global_accuracy)
@@ -188,108 +197,103 @@ def one_ir_mode(args):
def two_ir_mode(args):
core = get_plugin(args.device, args.l, args.config)
ref_core = get_plugin(args.reference_device, args.l, args.reference_config)
net = get_net(model=args.model, core=core)
net_layers, net_inputs, net_outputs = get_model_info(net)
ref_net = get_net(model=args.reference_model, core=ref_core)
ref_net_layers, ref_net_inputs, ref_net_outputs = get_model_info(ref_net)
model = get_model(model_path=args.model, core=core)
model_ops, model_inputs, model_outputs = get_model_info(model)
ref_model = get_model(model_path=args.reference_model, core=ref_core)
ref_model_ops, _, _ = get_model_info(ref_model)
check_inputs_and_default_outputs_are_equal(model, ref_model)
log.info(f'{args.device} vs {args.reference_device}')
log.info(f'IR for {args.device} : {args.model}')
log.info(f'IR for {args.reference_device} : {args.reference_model}')
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
ref_out_layers = get_layers_list(ref_net_layers, ref_net_inputs, ref_net_outputs, args.layers)
print_input_layers(net_inputs)
print_output_layers(out_layers)
layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping,
user_layers=out_layers)
inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input,
layers_map=layers_map)
ref_inputs = input_processing(model_path=args.reference_model, net_inputs=ref_net_inputs, input_file=args.input,
layers_map=layers_map)
if args.reference_layers:
out_ops = get_ops_list(model_ops, model_outputs, args.layers)
ref_out_ops = get_ops_list(ref_model_ops, model_outputs, args.reference_layers)
if len(out_ops) != len(ref_out_ops):
raise Exception("Number of layers to compare against should be equal!")
else:
ref_out_ops = out_ops = get_ops_list(get_ops_intersection(model_ops, ref_model_ops), model_outputs, args.layers)
print_inputs(model_inputs)
print_output_ops(get_ops_union(out_ops, ref_out_ops))
inputs = input_processing(model_path=args.model, model_inputs=model_inputs, input_file=args.input)
global_accuracy = []
global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.reference_model,
out_layers=out_layers, ref_out_layers=ref_out_layers,
inputs=inputs, ref_inputs=ref_inputs, core=core,
out_ops=out_ops, ref_out_ops=out_ops,
inputs=inputs, ref_inputs=inputs, core=core,
device=args.device, ref_core=ref_core,
ref_device=args.reference_device, layers=args.layers,
num_of_iterations=args.num_of_iterations)
for out_layer in layers_map:
ref_out_layer = layers_map[out_layer]
if out_layer == ref_out_layer:
log.info(f'Layer {out_layer} statistics')
for op, ref_op in zip(out_ops, ref_out_ops):
if op.friendly_name == ref_op.friendly_name:
log.info(f'Layer {op.friendly_name} statistics')
else:
log.info(f'Statistics \'{out_layer}\' vs \'{ref_out_layer}\'')
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
ref_net_copy = get_net_copy_with_output(model=args.reference_model, output=ref_out_layer, core=ref_core)
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
if out_layer not in results:
continue
out_blob, pc = results[out_layer]
ref_results = infer(net=ref_net_copy, core=ref_core, device=args.reference_device,
inputs=ref_inputs, output=[ref_out_layer])
ref_out_blob, ref_pc = ref_results[ref_out_layer]
if ref_out_layer not in ref_results:
continue
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
performance_metrics(pc=pc, ref_pc=ref_pc)
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
if op.get_output_size() != ref_op.get_output_size():
log.warning(f"Skipping {op.friendly_name} vs {ref_op.frinedly_name} comparison due to different number of outputs!")
continue
log.info(f'Layer {op.friendly_name} vs {ref_op.friendly_name} statistics')
for i in range(op.get_output_size()):
if op.get_output_size() > 1:
log.info(f'Port {i}: ')
model_copy, new_output = get_model_copy_with_output(model=args.model, output=(op.friendly_name, i), core=core)
ref_model_copy, ref_new_output = get_model_copy_with_output(model=args.reference_model, output=(ref_op.friendly_name, i), core=ref_core)
out_tensor, pc = infer(model=model_copy, core=core, device=args.device, inputs=inputs, output=new_output)
ref_out_tensor, ref_pc = infer(model=ref_model_copy, core=ref_core, device=args.reference_device,
inputs=inputs, output=ref_new_output)
a_m = accuracy_metrics(out_tensor, ref_out_tensor)
performance_metrics(args.device, pc, args.reference_device, ref_pc)
tensor_counters(out_tensor, ref_out_tensor)
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times,
global_accuracy=global_accuracy)
def dump_mode(args):
core = get_plugin(args.device, args.l, args.config)
net = get_net(model=args.model, core=core)
func = ng.function_from_cnn(net)
ops = func.get_ops()
out_layers = get_layers_list(ops, net.input_info, net.outputs, args.layers)
inputs = input_processing(args.model, net.input_info, args.input)
dump_dict = {}
for out_layer in out_layers:
log.info(f'Layer {out_layer} processing')
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
if out_layer not in results:
continue
out_blob, pc = results[out_layer]
dump_dict[out_layer] = np.array({'blob': out_blob, 'pc': pc})
model = get_model(model_path=args.model, core=core)
model_ops, model_inputs, model_outputs = get_model_info(model)
out_ops = get_ops_list(model_ops, model_outputs, args.layers)
inputs = input_processing(args.model, model_inputs, args.input)
dump_dict = defaultdict(list)
for op in out_ops:
for i in range(op.get_output_size()):
if op.get_output_size() > 1:
log.info(f'Layer {op.friendly_name}, port {i} processing')
else:
log.info(f'Layer {op.friendly_name} processing')
model_copy, new_output = get_model_copy_with_output(model=args.model, output=(op.friendly_name, i), core=core)
out_tensor, pc = infer(model=model_copy, core=core, device=args.device, inputs=inputs, output=new_output)
dump_dict[op.friendly_name].append(np.array({'tensor': out_tensor, 'pc': perf_counts_to_dump(pc)}))
dump_dict["device"] = args.device
dump_output_file(args.model + '_' + args.device + '_dump.npz', dump_dict)
def load_mode(args):
core = get_plugin(args.device, args.l, args.config)
log.info(f'IR for {args.device} : {args.model}')
log.info(f'Loading blob from {args.load}')
net = get_net(model=args.model, core=core)
net_layers, net_inputs, net_outputs = get_model_info(net)
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
print_input_layers(net_inputs)
print_output_layers(out_layers)
layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping,
user_layers=out_layers)
inputs = input_processing(args.model, net_inputs, args.input, layers_map)
log.info(f'Loading tensors from {args.load}')
model = get_model(model_path=args.model, core=core)
model_ops, model_inputs, model_outputs = get_model_info(model)
out_ops = get_ops_list(model_ops, model_outputs, args.layers)
print_inputs(model_inputs)
print_output_ops(out_ops)
inputs = input_processing(args.model, model_inputs, args.input)
global_accuracy = []
loaded = load_dump(args.load)
for out_layer in layers_map:
ref_out_layer = layers_map[out_layer]
if out_layer == ref_out_layer:
log.info(f'Layer {out_layer} statistics')
for op in out_ops:
if op.friendly_name in loaded:
log.info(f'Layer {op.friendly_name} statistics')
else:
log.info(f'Statistics \'{out_layer}\' vs \'{ref_out_layer}\'')
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
if out_layer not in results:
log.info(f'Statistics for layer \'{op.friendly_name}\' was not dumped. Skipping this layer.')
continue
out_blob, pc = results[out_layer]
if ref_out_layer not in loaded:
continue
ref_out_blob = loaded[ref_out_layer]['blob']
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
if 'pc' in loaded[ref_out_layer]:
ref_pc = loaded[ref_out_layer]['pc']
performance_metrics(pc=pc, ref_pc=ref_pc)
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
for i in range(op.get_output_size()):
if op.get_output_size() > 1:
log.info(f'Port {i}: ')
model_copy, new_output = get_model_copy_with_output(model=args.model, output=(op.friendly_name, i), core=core)
out_tensor, pc = infer(model=model_copy, core=core, device=args.device, inputs=inputs, output=new_output)
ref_out_tensor, ref_pc = loaded[op.friendly_name][i]['tensor'], load_profiling_info(loaded[op.friendly_name][i]['pc'])
a_m = accuracy_metrics(out_tensor, ref_out_tensor)
performance_metrics(args.device, pc, loaded["device"], ref_pc)
tensor_counters(out_tensor, ref_out_tensor)
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_accuracy=global_accuracy)
@@ -297,7 +301,7 @@ def main():
set_logger(log.DEBUG)
args = validate_args(build_parser().parse_args())
log.info(f'Inference Engine:\n API version ............ {ie.__version__}', extra={'no_lvl': True})
log.info(f'OpenVINO:\n API version ............ {get_version()}', extra={'no_lvl': True})
set_verbosity(args.verbosity)
mode = find_out_cct_mode(args)
if mode == 1:

325
tools/cross_check_tool/openvino/tools/cross_check_tool/utils.py
View File

@@ -6,7 +6,10 @@ import logging as log
import os
import sys
import traceback
import xml
from pathlib import Path
from openvino.runtime import Layout, ProfilingInfo, Output
from openvino.runtime.utils.types import get_dtype
try:
import cv2
@@ -137,7 +140,7 @@ def build_parser():
'\n--device device_for_model \\'
'\n--reference_device reference_device_for_model \n'
+ '-' * 62 +
'\nFor dumping blob and performance counters run:'
'\nFor dumping tensors and performance counters run:'
'\npython3 cross_check_tool.py \\'
'\n--input path/to/file/describing/input \\'
'\n--model path/to/model/*.xml \\'
@@ -162,7 +165,7 @@ def build_parser():
)
model = parser.add_argument_group('Model specific arguments')
model.add_argument('--input', '-i', type=str, action=ExistingFileAction,
model.add_argument('--input', '-i', type=str,
help='Path to an input image file or multi-input file to infer. Generates input(s) from normal '
'distribution if empty')
# model.add_argument('--batch', '-b', type=int, help='Overrides batch size. Default is inherited from model')
@@ -171,14 +174,13 @@ def build_parser():
model.add_argument('--reference_model', '-ref_m', type=str, action=ExistingFileAction,
help='Path to an .xml file that represents the second IR to compare the metrics. '
'Uses --model if empty')
# TODO: allow to pass layer type
model.add_argument('--layers', '-layers', type=str, default=None,
help='Defines layers to check. Options: all, None - for output layers check, list of '
'comma-separated layer names to check. Default value is None.')
model.add_argument('--mapping', '-map', type=str, action=ExistingFileAction,
help='Model Optimizer provided mapping for --model/-m')
model.add_argument('--reference_mapping', '-ref_map', type=str, action=ExistingFileAction,
help='Model Optimizer provided mapping for --reference_model/-ref_model')
model.add_argument('-ref_layers', '--reference_layers', type=str, default=None,
help='Defines layers to check in referece model. Options: all, None - for output layers check, list of '
'comma-separated layer names to check. If not specified the same layers will be processed as in --layers parameter.')
plugin = parser.add_argument_group('Plugin specific arguments')
plugin.add_argument('--plugin_path', '-pp', type=str, action=ExistingDirAction, help='Path to a plugin folder.')
plugin.add_argument('--device', '-d', type=str, required=True,
@@ -197,8 +199,8 @@ def build_parser():
modes = parser.add_argument_group('CCT mode arguments')
# TODO eps? nobody uses it
modes.add_argument('--dump', help='Enables blobs statistics dumping', action='store_true', default=False)
modes.add_argument('--load', type=str, action=ExistingFileAction, help='Path to a file to load blobs from')
modes.add_argument('--dump', help='Enables tensors statistics dumping', action='store_true', default=False)
modes.add_argument('--load', type=str, action=ExistingFileAction, help='Path to a file to load tensors from')
model.add_argument('--num_of_iterations', '-ni', type=int, default=50,
help='Number of iterations to collect all over the net performance')
parser.add_argument('-v', '--verbosity', action='store_true', default=False,
@@ -219,10 +221,10 @@ def validate_args(args):
args.model = args.reference_model
if args.model == args.reference_model:
args.reference_model = None
if args.model != args.reference_model and args.reference_model is not None and args.mapping is None and \
args.reference_mapping is None:
log.warning('Check over two different IRs was enabled. In case if layer names in this two IRs differ, '
'please provide mapping files with --mapping/-map and --reference_mapping/-ref_map')
if args.model != args.reference_model and args.reference_model is not None and (args.layers is None or \
args.reference_layers is None):
log.warning('Check over two different IRs was enabled. In case if layer names in these two IRs are different, '
'please provide both -layers and --reference_layers to compare against.')
# device check
if args.device is None and args.reference_device is None:
raise Exception("Parameters -device/-d and -reference_device/-ref_d are not set. Can not proceed."
@@ -273,14 +275,13 @@ def find_out_cct_mode(args):
raise Exception('Unknown Cross Check Tool CLI configuration.\nFor more details use -h option')
def print_input_layers(inputs: list):
def print_inputs(inputs: list):
word = 'inputs' if len(inputs) > 1 else 'input'
log.info(f"{len(inputs)} {word} detected: {', '.join(inputs)}")
log.info(f"{len(inputs)} {word} detected: {', '.join(input.any_name for input in inputs)}")
def print_output_layers(outputs: list):
layers = 'layers' if len(outputs) > 1 else 'layer'
log.info(f"Statistics will be dumped for {len(outputs)} {layers}: {', '.join(outputs)}")
def print_output_ops(output_ops: list):
layers = 'layers' if len(output_ops) > 1 else 'layer'
log.info(f"Statistics will be dumped for {len(output_ops)} {layers}: {', '.join(op.friendly_name for op in output_ops)}")
###
@@ -306,70 +307,132 @@ def get_config_dictionary(config_file):
###
def read_multi_input_file(input_file: str, net_inputs: dict):
def read_multi_input_file(input_file: str, model_inputs: list):
npz = np.load(input_file, allow_pickle=True)
files = npz.files
dump = {}
for net_input in net_inputs:
if net_input not in files:
raise Exception(f"Can not find input data for input {net_input} in multi-input file {input_file}.\n"
dump = []
for model_input in model_inputs:
if model_input.any_name not in files:
raise Exception(f"Can not find input data for input {model_input.any_name} in multi-input file {input_file}.\n"
f"Input data was provided for layers: {', '.join(files)}\n"
f"Network inputs: {', '.join(net_inputs.keys())}")
if 'blob' in npz[net_input].item(0):
just_blob = npz[net_input].item(0)['blob']
network_shape = net_inputs[net_input].input_data.shape
log.info(f'Layer {net_input} shape = {network_shape}, input blob from multi-input file shape = {just_blob.shape}')
f"Network inputs: {', '.join(input.any_name for input in model_inputs)}")
if 'tensor' in npz[model_input.any_name].item(0):
just_tensor = npz[model_input.any_name].item(0)['tensor']
input_shape = list(model_input.shape)
log.info(f'Layer {model_input.any_name} shape = {input_shape}, input tensor from multi-input file shape = {just_tensor.shape}')
try:
reshaped_blob = np.reshape(just_blob, network_shape)
reshaped_tensor = np.reshape(just_tensor, input_shape)
except:
raise Exception(f'Can not reshape input blob from multi-input file for layer {net_input} to shape {network_shape}')
dump[net_input] = reshaped_blob
raise Exception(f'Can not reshape input tensor from multi-input file for layer {model_input.any_name} to shape {input_shape}')
dump.append(reshaped_tensor)
else:
raise Exception(
f'Can not find \'blob\' parameter for input {net_input} in input file {input_file}')
f'Can not find \'tensor\' parameter for input {model_input.any_name} in input file {input_file}')
return dump
def is_chw_input(input):
if input.node.layout == Layout("NCHW") or input.node.layout == Layout("CHW"):
return True
shape = input.shape
if len(shape) == 4:
return shape[1] == 3
if len(shape) == 3:
return shape[0] == 3
return False
def get_input_sizes(input):
if is_chw_input(input):
shape = input.shape
if len(shape) == 3:
return shape[1], shape[2]
else:
return shape[2], shape[3]
else:
shape = input.shape
if len(shape) < 3 or len(shape) > 4:
raise Exception('Can not interpret input shape as image')
if len(shape) == 3:
return shape[0], shape[1]
else:
return shape[1], shape[2]
@error_handling('reading --input/-i by OpenCV python module. OpenCV version: {}. '
'It may happen due to wrong input image format'.format(cv2.__version__))
def read_image_file(input_file: str, net_inputs: dict):
inputs = dict()
if len(net_inputs) == 1:
image = cv2.imread(input_file)
if image is None:
raise Exception('Can not read input image ' + input_file)
only_layer_name = list(net_inputs.keys())[0]
shape = net_inputs[only_layer_name].input_data.shape
if len(shape) != 4:
raise Exception('Can not interpret input shape as image')
n, c, h, w = shape
image = cv2.resize(image, (w, h))
def read_image_file(image_file: str, model_input: Output):
image_file = str(image_file)
log.info(f'Prepare image {image_file}')
image = cv2.imread(image_file)
if image is None:
raise Exception('Can not read input image ' + image_file)
h, w = get_input_sizes(model_input)
image = cv2.resize(image, (w, h))
if is_chw_input(model_input):
image = image.transpose((2, 0, 1)) # Change data layout from HWC to CHW
image = image.reshape((n, c, h, w))
inputs[only_layer_name] = image
else:
raise Exception('Multi-input topology detected. Please provide multi-input file to --input key')
if len(model_input.shape) == 4:
image = np.expand_dims(image, 0) # Add batch dimension
return image
def get_random_inputs(model_inputs, model_path):
inputs = [np.clip(np.random.normal(0.5, 0.1, size=list(input.shape)), 0, 1) for input in model_inputs]
dump_output_file(model_path + '_random_input_dump.npz', {model_inputs[i].any_name: {'tensor': inputs[i]} for i in range(len(model_inputs))})
return inputs
def input_processing(model_path: str, net_inputs: dict, input_file: str, layers_map: dict = None):
inputs = dict()
def read_binary_file(bin_file, model_input):
log.info(f"Prepare binary file {str(bin_file)}")
binary_file_size = os.path.getsize(bin_file)
tensor_size = model_input.tensor.size
if tensor_size != binary_file_size:
raise Exception(f"File {bin_file} contains {binary_file_size} bytes but model expects {tensor_size}")
return np.reshape(np.fromfile(bin_file, get_dtype(model_input.element_type)), list(model_input.shape))
def input_processing(model_path: str, model_inputs: list, input_file: str):
if input_file is None:
for net_input in net_inputs:
inputs[net_input] = np.clip(np.random.normal(0.5, 0.1, size=net_inputs[net_input].input_data.shape), 0, 1)
dump_output_file(model_path + '_random_input_dump.npz', {inp: {'blob': inputs[inp]} for inp in inputs})
return inputs
try:
inputs = read_multi_input_file(input_file=input_file, net_inputs=net_inputs)
except:
inputs = read_image_file(input_file=input_file, net_inputs=net_inputs)
return inputs
return get_random_inputs(model_inputs, model_path)
else:
inputs = input_file.split(',')
input_names = [input.any_name for input in model_inputs]
input_data = {}
for i in range(min(len(inputs), len(model_inputs))):
splited = inputs[i].rsplit(':', maxsplit=1)
if len(splited) == 0:
raise Exception(f"Can't parse {'input_file'} input parameter!")
tensor_name = None
if len(splited) == 1:
tensor_name = input_names[i]
else:
tensor_name = splited.pop(0)
if tensor_name not in input_names:
raise Exception(f"Input with name {tensor_name} doesn't exist in the model!")
path = Path(splited.pop())
if path.exists() and path.is_file():
IMAGE_EXTENSIONS = ['.jpeg', '.jpg', '.png', '.bmp']
BINARY_EXTENSIONS = ['.bin']
NUMPY_EXTENSIONS = ['.npy', '.npz']
current_input = model_inputs[input_names.index(tensor_name)]
if path.suffix.lower() in IMAGE_EXTENSIONS:
input_data[tensor_name] = read_image_file(path, current_input)
elif path.suffix.lower() in BINARY_EXTENSIONS:
input_data[tensor_name] = read_binary_file(path, current_input)
elif path.suffix.lower() in NUMPY_EXTENSIONS:
return read_multi_input_file(path, model_inputs)
elif path.is_dir():
raise Exception(f"Path `{path}` is a directory! Provide full path to an input file!")
elif not path.exists():
raise Exception(f"Input file `{path}` doesn't exist!")
return input_data
def accuracy_metrics(out_blob, ref_out_blob):
if out_blob.size != ref_out_blob.size:
raise Exception(f'Different number of elements in blobs {out_blob.size} and {ref_out_blob.size}. Can not compare')
abs_diff = np.absolute(out_blob - ref_out_blob)
def accuracy_metrics(tensor, ref_tensor):
if tensor.size != ref_tensor.size:
raise Exception(f'Different number of elements in tensors {tensor.size} and {ref_tensor.size}. Can not compare')
abs_diff = np.absolute(tensor - ref_tensor)
np.seterr(divide='ignore', invalid='ignore')
rel_diff = np.divide(abs_diff, np.min(abs_diff) if np.min(abs_diff) != 0 else 1e-20)
metrics = [
@@ -377,14 +440,14 @@ def accuracy_metrics(out_blob, ref_out_blob):
('Min absolute difference', np.min(abs_diff)),
('Max relative difference', np.max(rel_diff)),
('Min relative difference', np.min(rel_diff)),
('Min reference value', np.min(ref_out_blob)),
('Min absolute reference value', np.min(np.abs(ref_out_blob))),
('Max reference value', np.max(ref_out_blob)),
('Max absolute reference value', np.max(np.abs(ref_out_blob))),
('Min actual value', np.min(out_blob)),
('Min absolute actual value', np.min(np.abs(out_blob))),
('Max actual value', np.max(out_blob)),
('Max absolute actual value', np.max(np.abs(out_blob)))
('Min reference value', np.min(ref_tensor)),
('Min absolute reference value', np.min(np.abs(ref_tensor))),
('Max reference value', np.max(ref_tensor)),
('Max absolute reference value', np.max(np.abs(ref_tensor))),
('Min actual value', np.min(tensor)),
('Min absolute actual value', np.min(np.abs(tensor))),
('Max actual value', np.max(tensor)),
('Max absolute actual value', np.max(np.abs(tensor)))
]
for key, value in metrics:
@@ -395,24 +458,24 @@ def accuracy_metrics(out_blob, ref_out_blob):
return {metric: value for metric, value in metrics}
def performance_metrics(pc, ref_pc):
def performance_metrics(device, pc, ref_device, ref_pc):
compare = [
('Device', '-d ' + pc['device'], '-ref_d ' + ref_pc['device']),
('Status', pc['status'], ref_pc['status']),
('Layer type', pc['layer_type'], ref_pc['layer_type']),
('Real time, microsec', pc['real_time'], ref_pc['real_time'])
('Device', '-d ' + device, '-ref_d ' + ref_device),
('Status', str(pc.status), str(ref_pc.status)),
('Layer type', pc.node_type, ref_pc.node_type),
('Real time, microsec', str(pc.real_time), str(ref_pc.real_time))
]
for metric, actual, reference in compare:
log.info(f'{metric:>35}: {actual:>16} {reference:>16}', extra={'no_lvl': True})
def blob_counters(out_blob, ref_out_blob):
def tensor_counters(tensor, ref_tensor):
counters = [
('Number of NAN', np.sum(np.isnan(out_blob)), np.sum(np.isnan(ref_out_blob))),
('Number of INF', np.sum(np.isinf(out_blob)), np.sum(np.isinf(ref_out_blob))),
('Number of ZERO', out_blob.size - np.count_nonzero(out_blob),
ref_out_blob.size - np.count_nonzero(ref_out_blob))
('Number of NAN', np.sum(np.isnan(tensor)), np.sum(np.isnan(ref_tensor))),
('Number of INF', np.sum(np.isinf(tensor)), np.sum(np.isinf(ref_tensor))),
('Number of ZERO', tensor.size - np.count_nonzero(tensor),
ref_tensor.size - np.count_nonzero(ref_tensor))
]
for metric, actual, reference in counters:
log.info(f'{metric:>35}: {actual:>16} {reference:>16}', extra={'no_lvl': True})
@@ -435,7 +498,7 @@ def update_global_accuracy_matrics(global_accuracy: list, current_accuracy: dict
return global_accuracy
def print_all_over_the_net_metrics(global_accuracy: (str, float), global_times: list = None,
def print_all_over_the_net_metrics(global_accuracy: list, global_times: list = None,
ref_global_times: list = None):
if global_times is not None and ref_global_times is not None and len(global_times) and len(ref_global_times):
log.info('-' * 70, extra={'no_lvl': True})
@@ -453,74 +516,39 @@ def print_all_over_the_net_metrics(global_accuracy: (str, float), global_times:
###
def read_mapping(file_name: str):
# TODO check twice
mapping_dict = {}
xml_tree = xml.etree.ElementTree.parse(file_name)
xml_root = xml_tree.getroot()
for child in xml_root:
fw_info = child.find('.//framework')
ir_info = child.find('.//IR')
if fw_info is None:
continue
if ir_info is None:
continue
framework_name = fw_info.attrib['name'] + ':' + fw_info.attrib['out_port_id']
ir_name = ir_info.attrib['name'] if ir_info is not None else None
ir_layer_id = int(ir_info.attrib['id']) if ir_info is not None else None
mapping_dict[framework_name] = (ir_name, ir_layer_id)
return mapping_dict
def map_layers(mapping_file: str = None, ref_mapping_file: str = None):
if mapping_file is not None and ref_mapping_file is not None:
mapping = read_mapping(mapping_file)
ref_mapping = read_mapping(ref_mapping_file)
mapping = {layer: ref_layer for layer in mapping for ref_layer in ref_mapping if layer == ref_layer}
return mapping
def manage_user_outputs_with_mapping(mapping, reference_mapping, user_layers):
if mapping is not None and reference_mapping is not None:
layers_map = map_layers(mapping, reference_mapping)
else:
layers_map = {layer: layer for layer in user_layers}
for layer in user_layers:
if layer not in layers_map:
if mapping is not None and reference_mapping is not None:
log.warning(
f'Can not map layer {layer} from --model/-m to any layer from --reference_model/-ref_m')
else:
log.warning(f'Can not find layer {layer} in --reference_model/-ref_m model')
for layer in layers_map:
if layer not in user_layers:
del layers_map[layer]
return layers_map
def get_layers_list(all_layers: list, inputs: dict, outputs: list, layers: str):
def get_ops_list(all_ops: list, outputs: list, layers: str):
if layers is not None and layers != 'None':
if layers == 'all':
return {layer.get_friendly_name(): layer for layer in all_layers \
if layer.get_type_name() not in ['Constant', 'Result']}
return [op for op in all_ops if op.get_type_name() not in ['Constant', 'Result', 'Parameter']]
else:
all_layers_names = {op.get_friendly_name() : op for op in all_layers}
user_layers = [layer.strip() for layer in layers.split(',')]
layers_to_check = []
for user_layer in user_layers:
if user_layer not in all_layers_names:
raise Exception(f"Layer {user_layer} doesn't exist in the model")
if user_layer in inputs:
raise Exception(f"Layer {user_layer} is input layer. Can not proceed")
if all_layers_names[user_layer].get_type_name() != 'Result':
layers_to_check.append(user_layer)
all_ops_map = {op.friendly_name: op for op in all_ops}
user_ops = [layer.strip() for layer in layers.split(',')]
new_outputs = []
for user_op in user_ops:
if user_op not in all_ops_map:
raise Exception(f"Operation with name `{user_op}` doesn't exist in the model")
else:
# if layer type is Result - add previous layer
prev_layer = all_layers[len(all_layers)-2]
layers_to_check.append(prev_layer.get_friendly_name())
return layers_to_check
new_outputs.append(all_ops_map[user_op])
return new_outputs
else:
return outputs
return [output.node for output in outputs]
def perf_counts_to_dump(prof_info):
perf_count = {}
perf_count["status"] = prof_info.status
perf_count["real_time"] = prof_info.real_time
perf_count["cpu_time"] = prof_info.cpu_time
perf_count["exec_type"] = prof_info.exec_type
perf_count["node_type"] = prof_info.node_type
return perf_count
def load_profiling_info(pi_dumped):
prof_info = ProfilingInfo()
for property, value in pi_dumped.items():
setattr(prof_info, property, value)
return prof_info
###
@@ -534,5 +562,6 @@ def dump_output_file(output_file, dump_dict):
def load_dump(file_to_load: str):
npz = np.load(file_to_load, allow_pickle=True)
dump = {file: npz[file].item(0) for file in npz}
dump = {file: [npz[file].item(i).item(0) for i in range(npz[file].size)] for file in npz if file != "device"}
dump["device"] = npz["device"].item(0)
return dump