IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[PyOV] Fix models checking and ensure correct destructor calls in tests (#16814)

Browse Source
This commit is contained in:
Jan Iwaszkiewicz
2023-04-13 10:37:05 +02:00
committed by GitHub
parent ae350c7107
commit 63c5be3ed2
5 changed files with 152 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
src/bindings/python/tests/test_graph/test_manager.py
View File

@@ -11,9 +11,9 @@ import pytest
import openvino.runtime.opset10 as ops
from openvino.runtime import Core, Model
from openvino.runtime.passes import Manager, Serialize, ConstantFolding, Version
from tests.test_graph.util import count_ops_of_type
from tests.test_utils.test_utils import create_filename_for_test
from tests.test_graph.util import count_ops_of_type
from tests.test_utils.test_utils import create_filename_for_test, compare_models
def create_model():
shape = [100, 100, 2]
@@ -68,18 +68,17 @@ def test_serialize_separate_paths_kwargs(prepare_ir_paths):
parameter_a = ops.parameter(shape, dtype=np.float32, name="A")
parameter_b = ops.parameter(shape, dtype=np.float32, name="B")
parameter_c = ops.parameter(shape, dtype=np.float32, name="C")
model = (parameter_a + parameter_b) * parameter_c
func = Model(model, [parameter_a, parameter_b, parameter_c], "Model")
_model = (parameter_a + parameter_b) * parameter_c
model = Model(_model, [parameter_a, parameter_b, parameter_c], "Model")
xml_path, bin_path = prepare_ir_paths
pass_manager = Manager()
pass_manager.register_pass(Serialize(path_to_xml=xml_path, path_to_bin=bin_path))
pass_manager.run_passes(func)
pass_manager.run_passes(model)
res_model = core.read_model(model=xml_path, weights=bin_path)
assert func.get_parameters() == res_model.get_parameters()
assert func.get_ordered_ops() == res_model.get_ordered_ops()
assert compare_models(model, res_model)
def test_serialize_separate_paths_args(prepare_ir_paths):
@@ -90,18 +89,17 @@ def test_serialize_separate_paths_args(prepare_ir_paths):
parameter_b = ops.parameter(shape, dtype=np.float32, name="B")
parameter_c = ops.parameter(shape, dtype=np.float32, name="C")
parameter_d = ops.parameter(shape, dtype=np.float32, name="D")
model = ((parameter_a + parameter_b) * parameter_c) / parameter_d
func = Model(model, [parameter_a, parameter_b, parameter_c, parameter_d], "Model")
_model = ((parameter_a + parameter_b) * parameter_c) / parameter_d
model = Model(_model, [parameter_a, parameter_b, parameter_c, parameter_d], "Model")
xml_path, bin_path = prepare_ir_paths
pass_manager = Manager()
pass_manager.register_pass(Serialize(xml_path, bin_path))
pass_manager.run_passes(func)
pass_manager.run_passes(model)
res_model = core.read_model(model=xml_path, weights=bin_path)
assert func.get_parameters() == res_model.get_parameters()
assert func.get_ordered_ops() == res_model.get_ordered_ops()
assert compare_models(model, res_model)
def test_serialize_pass_mixed_args_kwargs(prepare_ir_paths):
@@ -110,18 +108,17 @@ def test_serialize_pass_mixed_args_kwargs(prepare_ir_paths):
shape = [3, 2]
parameter_a = ops.parameter(shape, dtype=np.float32, name="A")
parameter_b = ops.parameter(shape, dtype=np.float32, name="B")
model = parameter_a - parameter_b
func = Model(model, [parameter_a, parameter_b], "Model")
_model = parameter_a - parameter_b
model = Model(_model, [parameter_a, parameter_b], "Model")
xml_path, bin_path = prepare_ir_paths
pass_manager = Manager()
pass_manager.register_pass(Serialize(xml_path, path_to_bin=bin_path))
pass_manager.run_passes(func)
pass_manager.run_passes(model)
res_model = core.read_model(model=xml_path, weights=bin_path)
assert func.get_parameters() == res_model.get_parameters()
assert func.get_ordered_ops() == res_model.get_ordered_ops()
assert compare_models(model, res_model)
def test_serialize_pass_mixed_args_kwargs_v2(prepare_ir_paths):
@@ -135,8 +132,7 @@ def test_serialize_pass_mixed_args_kwargs_v2(prepare_ir_paths):
res_model = core.read_model(model=xml_path, weights=bin_path)
assert model.get_parameters() == res_model.get_parameters()
assert model.get_ordered_ops() == res_model.get_ordered_ops()
assert compare_models(model, res_model)
# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request
@@ -153,39 +149,16 @@ def test_serialize_results(prepare_ir_paths):
core = Core()
node_constant = ops.constant(np.array([[0.0, 0.1, -0.1], [-2.5, 2.5, 3.0]], dtype=np.float32))
node_ceil = ops.ceiling(node_constant)
func = Model(node_ceil, [], "Model")
model = Model(node_ceil, [], "Model")
xml_path, bin_path = prepare_ir_paths
pass_manager = Manager()
pass_manager.register_pass(Serialize(path_to_xml=xml_path, path_to_bin=bin_path))
pass_manager.run_passes(func)
pass_manager.run_passes(model)
res_model = core.read_model(model=xml_path, weights=bin_path)
const = func.get_results()[0].input(0).get_source_output().get_node()
new_const = res_model.get_results()[0].input(0).get_source_output().get_node()
assert const == new_const
import gc
gc.collect() # CVS-106805
@pytest.mark.xfail(reason="CVS-106805. Delete test after fix")
def test_serialize_results_fails(prepare_ir_paths):
core = Core()
node_constant = ops.constant(np.array([[0.0, 0.1, -0.1], [-2.5, 2.5, 3.0]], dtype=np.float32))
func = Model(node_constant, [], "Model")
xml_path, bin_path = prepare_ir_paths
pass_manager = Manager()
pass_manager.register_pass(Serialize(path_to_xml=xml_path, path_to_bin=bin_path))
pass_manager.run_passes(func)
res_model = core.read_model(model=xml_path, weights=bin_path)
const = func.get_results()
new_const = res_model.get_results()
assert const == new_const
assert compare_models(model, res_model)
def test_default_version(prepare_ir_paths):
@@ -199,8 +172,7 @@ def test_default_version(prepare_ir_paths):
res_model = core.read_model(model=xml_path, weights=bin_path)
assert model.get_parameters() == res_model.get_parameters()
assert model.get_ordered_ops() == res_model.get_ordered_ops()
assert compare_models(model, res_model)
def test_default_version_IR_V11_seperate_paths(prepare_ir_paths):
@@ -214,5 +186,4 @@ def test_default_version_IR_V11_seperate_paths(prepare_ir_paths):
res_model = core.read_model(model=xml_path, weights=bin_path)
assert model.get_parameters() == res_model.get_parameters()
assert model.get_ordered_ops() == res_model.get_ordered_ops()
assert compare_models(model, res_model)

5
src/bindings/python/tests/test_runtime/test_core.py
View File

@@ -30,6 +30,7 @@ from tests.test_utils.test_utils import (
generate_relu_compiled_model,
get_relu_model,
plugins_path,
compare_models,
)
@@ -158,9 +159,7 @@ def test_model_from_buffer_valid():
xml = f.read()
model = core.read_model(model=xml, weights=weights)
ref_model = core.read_model(model=test_net_xml, weights=test_net_bin)
assert model.get_parameters() == ref_model.get_parameters()
assert model.get_results() == ref_model.get_results()
assert model.get_ordered_ops() == ref_model.get_ordered_ops()
assert compare_models(model, ref_model)
def test_get_version(device):

51
src/bindings/python/tests/test_transformations/test_offline_api.py
View File

@@ -5,7 +5,6 @@
import os
import pytest
import numpy as np
from openvino.runtime import serialize
from openvino._offline_transformations import (
apply_moc_transformations,
apply_pot_transformations,
@@ -17,10 +16,10 @@ from openvino._offline_transformations import (
apply_fused_names_cleanup,
)
from openvino.runtime import Model, PartialShape, Core
from openvino.runtime import Model, PartialShape, Core, serialize
import openvino.runtime as ov
from tests.test_utils.test_utils import create_filename_for_test
from tests.test_utils.test_utils import create_filename_for_test, compare_models
def get_relu_model():
@@ -183,17 +182,18 @@ def test_serialize_pass_v2(request, tmp_path, is_path_xml, is_path_bin):
shape = [100, 100, 2]
parameter_a = ov.opset8.parameter(shape, dtype=np.float32, name="A")
parameter_b = ov.opset8.parameter(shape, dtype=np.float32, name="B")
model = ov.opset8.floor(ov.opset8.minimum(ov.opset8.abs(parameter_a), parameter_b))
func = Model(model, [parameter_a, parameter_b], "Model")
_model = ov.opset8.floor(ov.opset8.minimum(ov.opset8.abs(parameter_a), parameter_b))
model = Model(_model, [parameter_a, parameter_b], "Model")
serialize(func, xml_path, bin_path)
serialize(model, xml_path, bin_path)
assert func is not None
assert model is not None
res_model = core.read_model(model=xml_path, weights=bin_path)
assert func.get_parameters() == res_model.get_parameters()
assert func.get_ordered_ops() == res_model.get_ordered_ops()
assert compare_models(model, res_model)
del res_model
os.remove(xml_path)
os.remove(bin_path)
@@ -229,14 +229,15 @@ def test_version_default(request, tmp_path, is_path_xml, is_path_bin):
shape = [100, 100, 2]
parameter_a = ov.opset8.parameter(shape, dtype=np.float32, name="A")
parameter_b = ov.opset8.parameter(shape, dtype=np.float32, name="B")
model = ov.opset8.floor(ov.opset8.minimum(ov.opset8.abs(parameter_a), parameter_b))
func = Model(model, [parameter_a, parameter_b], "Model")
_model = ov.opset8.floor(ov.opset8.minimum(ov.opset8.abs(parameter_a), parameter_b))
model = Model(_model, [parameter_a, parameter_b], "Model")
serialize(func, xml_path, bin_path)
serialize(model, xml_path, bin_path)
res_model = core.read_model(model=xml_path, weights=bin_path)
assert func.get_parameters() == res_model.get_parameters()
assert func.get_ordered_ops() == res_model.get_ordered_ops()
assert compare_models(model, res_model)
del res_model
os.remove(xml_path)
os.remove(bin_path)
@@ -269,14 +270,15 @@ def test_version_ir_v10(request, tmp_path):
shape = [100, 100, 2]
parameter_a = ov.opset8.parameter(shape, dtype=np.float32, name="A")
parameter_b = ov.opset8.parameter(shape, dtype=np.float32, name="B")
model = ov.opset8.floor(ov.opset8.minimum(ov.opset8.abs(parameter_a), parameter_b))
func = Model(model, [parameter_a, parameter_b], "Model")
_model = ov.opset8.floor(ov.opset8.minimum(ov.opset8.abs(parameter_a), parameter_b))
model = Model(_model, [parameter_a, parameter_b], "Model")
serialize(func, xml_path, bin_path, "IR_V10")
serialize(model, xml_path, bin_path, "IR_V10")
res_model = core.read_model(model=xml_path, weights=bin_path)
assert func.get_parameters() == res_model.get_parameters()
assert func.get_ordered_ops() == res_model.get_ordered_ops()
assert compare_models(model, res_model, compare_names=False)
del res_model
os.remove(xml_path)
os.remove(bin_path)
@@ -289,14 +291,15 @@ def test_version_ir_v11(request, tmp_path):
shape = [100, 100, 2]
parameter_a = ov.opset8.parameter(shape, dtype=np.float32, name="A")
parameter_b = ov.opset8.parameter(shape, dtype=np.float32, name="B")
model = ov.opset8.floor(ov.opset8.minimum(ov.opset8.abs(parameter_a), parameter_b))
func = Model(model, [parameter_a, parameter_b], "Model")
_model = ov.opset8.floor(ov.opset8.minimum(ov.opset8.abs(parameter_a), parameter_b))
model = Model(_model, [parameter_a, parameter_b], "Model")
serialize(func, xml_path, bin_path, "IR_V11")
serialize(model, xml_path, bin_path, "IR_V11")
res_model = core.read_model(model=xml_path, weights=bin_path)
assert func.get_parameters() == res_model.get_parameters()
assert func.get_ordered_ops() == res_model.get_ordered_ops()
assert compare_models(model, res_model)
del res_model
os.remove(xml_path)
os.remove(bin_path)

16
src/bindings/python/tests/test_transformations/test_public_transformations.py
View File

@@ -15,8 +15,9 @@ from openvino.runtime.passes import (
LowLatency2,
Serialize,
)
from tests.test_transformations.utils.utils import count_ops, get_relu_model
from tests.test_utils.test_utils import create_filename_for_test
from tests.test_utils.test_utils import create_filename_for_test, compare_models
def get_model():
@@ -136,18 +137,19 @@ def test_serialize_pass(request, tmp_path, is_path_xml, is_path_bin):
is_path_xml,
is_path_bin)
func = get_relu_model()
model = get_relu_model()
manager = Manager()
manager.register_pass(Serialize(xml_path, bin_path))
manager.run_passes(func)
manager.run_passes(model)
assert func is not None
assert model is not None
res_func = core.read_model(model=xml_path, weights=bin_path)
res_model = core.read_model(model=xml_path, weights=bin_path)
assert func.get_parameters() == res_func.get_parameters()
assert func.get_ordered_ops() == res_func.get_ordered_ops()
assert compare_models(model, res_model)
del res_model
os.remove(xml_path)
os.remove(bin_path)

102
src/bindings/python/tests/test_utils/test_utils.py
View File

@@ -17,14 +17,100 @@ from openvino.runtime import Model, Core, Shape
from openvino.utils import deprecated
def test_compare_models():
try:
from openvino.test_utils import compare_models
model = get_relu_model()
status, _ = compare_models(model, model)
assert status
except RuntimeError:
print("openvino.test_utils.compare_models is not available") # noqa: T201
def _compare_models(model_one: Model, model_two: Model, compare_names: bool = True) -> Tuple[bool, str]: # noqa: C901 the function is too complex
"""Function to compare OpenVINO model (ops names, types and shapes).
Note that the functions uses get_ordered_ops, so the topological order of ops should be also preserved.
:param model_one: The first model to compare.
:param model_two: The second model to compare.
:param compare_names: Flag to control friendly names checking. Default: True
:return: Tuple which consists of bool value (True if models are equal, otherwise False)
and string with the message to reuse for debug/testing purposes. The string value
is empty when models are equal.
"""
result = True
msg = ""
# Check friendly names of models
if compare_names and model_one.get_friendly_name() != model_two.get_friendly_name():
result = False
msg += "Friendly names of models are not equal "
msg += f"model_one: {model_one.get_friendly_name()}, model_two: {model_two.get_friendly_name()}.\n"
model_one_ops = model_one.get_ordered_ops()
model_two_ops = model_two.get_ordered_ops()
# Check overall number of operators
if len(model_one_ops) != len(model_two_ops):
result = False
msg += "Not equal number of ops "
msg += f"model_one: {len(model_one_ops)}, model_two: {len(model_two_ops)}.\n"
for i in range(len(model_one_ops)):
op_one_name = model_one_ops[i].get_friendly_name() # op from model_one
op_two_name = model_two_ops[i].get_friendly_name() # op from model_two
# Check friendly names
if (compare_names and op_one_name != op_two_name and model_one_ops[i].get_type_name() != "Constant"):
result = False
msg += "Not equal op names "
msg += f"model_one: {op_one_name}, "
msg += f"model_two: {op_two_name}.\n"
# Check output sizes
if model_one_ops[i].get_output_size() != model_two_ops[i].get_output_size():
result = False
msg += f"Not equal output sizes of {op_one_name} and {op_two_name}.\n"
for idx in range(model_one_ops[i].get_output_size()):
# Check partial shapes of outputs
op_one_partial_shape = model_one_ops[i].get_output_partial_shape(idx)
op_two_partial_shape = model_two_ops[i].get_output_partial_shape(idx)
if op_one_partial_shape != op_two_partial_shape:
result = False
msg += f"Not equal op partial shapes of {op_one_name} and {op_two_name} on {idx} index "
msg += f"model_one: {op_one_partial_shape}, "
msg += f"model_two: {op_two_partial_shape}.\n"
# Check element types of outputs
op_one_element_type = model_one_ops[i].get_output_element_type(idx)
op_two_element_type = model_two_ops[i].get_output_element_type(idx)
if op_one_element_type != op_two_element_type:
result = False
msg += f"Not equal output element types of {op_one_name} and {op_two_name} on {idx} index "
msg += f"model_one: {op_one_element_type}, "
msg += f"model_two: {op_two_element_type}.\n"
return result, msg
def compare_models(model_one: Model, model_two: Model, compare_names: bool = True):
"""Function to compare OpenVINO model (ops names, types and shapes).
:param model_one: The first model to compare.
:param model_two: The second model to compare.
:param compare_names: Flag to control friendly names checking. Default: True
:return: True if models are equal, otherwise raise an error with a report of mismatches.
"""
result, msg = _compare_models(model_one, model_two, compare_names=compare_names)
if not result:
raise RuntimeError(msg)
return result
def test_compare_models_pass():
model = get_relu_model()
assert compare_models(model, model)
def test_compare_models_fail():
model = get_relu_model()
changed_model = model.clone()
changed_model.get_ordered_ops()[0].set_friendly_name("ABC")
with pytest.raises(RuntimeError) as e:
_ = compare_models(model, changed_model)
assert "Not equal op names model_one: data, model_two: ABC." in str(e.value)
def plugins_path(device, lib_path):