diff --git a/src/bindings/python/tests/conftest.py b/src/bindings/python/tests/conftest.py index 9583f8980ab..c091b4247e1 100644 --- a/src/bindings/python/tests/conftest.py +++ b/src/bindings/python/tests/conftest.py @@ -5,74 +5,15 @@ import os import pytest -from sys import platform -from openvino.runtime import Core - - -def get_model_with_template_extension(): - core = Core() - ir = bytes(b""" - - - - - - 1 - 3 - 22 - 22 - - - - - - - 1 - 3 - 22 - 22 - - - - - 1 - 3 - 22 - 22 - - - - - - - 1 - 3 - 22 - 22 - - - - - - - - -""") - if platform == "win32": - core.add_extension(library_path="openvino_template_extension.dll") - else: - core.add_extension(library_path="libopenvino_template_extension.so") - return core, core.read_model(ir) - def model_path(is_fp16=False): base_path = os.path.dirname(__file__) if is_fp16: - test_xml = os.path.join(base_path, "test_utils", "utils", "test_model_fp16.xml") - test_bin = os.path.join(base_path, "test_utils", "utils", "test_model_fp16.bin") + test_xml = os.path.join(base_path, "utils", "utils", "test_model_fp16.xml") + test_bin = os.path.join(base_path, "utils", "utils", "test_model_fp16.bin") else: - test_xml = os.path.join(base_path, "test_utils", "utils", "test_model_fp32.xml") - test_bin = os.path.join(base_path, "test_utils", "utils", "test_model_fp32.bin") + test_xml = os.path.join(base_path, "utils", "utils", "test_model_fp32.xml") + test_bin = os.path.join(base_path, "utils", "utils", "test_model_fp32.bin") return (test_xml, test_bin) diff --git a/src/bindings/python/tests/test_graph/test_manager.py b/src/bindings/python/tests/test_graph/test_manager.py index 7b42b62a219..30dfa210dc7 100644 --- a/src/bindings/python/tests/test_graph/test_manager.py +++ b/src/bindings/python/tests/test_graph/test_manager.py @@ -13,7 +13,7 @@ from openvino.runtime import Core, Model from openvino.runtime.passes import Manager, Serialize, ConstantFolding, Version from tests.test_graph.util import count_ops_of_type -from tests.test_utils.test_utils import create_filename_for_test, compare_models +from tests.utils.helpers import create_filename_for_test, compare_models def create_model(): shape = [100, 100, 2] diff --git a/src/bindings/python/tests/test_runtime/test_compiled_model.py b/src/bindings/python/tests/test_runtime/test_compiled_model.py index 1b9abc76a45..a977aea64ff 100644 --- a/src/bindings/python/tests/test_runtime/test_compiled_model.py +++ b/src/bindings/python/tests/test_runtime/test_compiled_model.py @@ -2,17 +2,18 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import os import pytest import numpy as np -from tests.conftest import model_path -from tests.test_utils.test_utils import get_relu_model, generate_image, generate_model_and_image, generate_relu_compiled_model -from openvino import Model, Shape, Core, Tensor +from tests.utils.helpers import ( + get_relu_model, + generate_image, + generate_model_and_image, + generate_relu_compiled_model, + create_filename_for_test) +from openvino import Model, Shape, Core, Tensor, serialize from openvino.runtime import ConstOutput -test_net_xml, test_net_bin = model_path() - def test_get_property(device): model = get_relu_model([1, 3, 32, 32]) @@ -23,9 +24,7 @@ def test_get_property(device): def test_get_runtime_model(device): - core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) - compiled_model = core.compile_model(model, device) + compiled_model = generate_relu_compiled_model(device) runtime_model = compiled_model.get_runtime_model() assert isinstance(runtime_model, Model) @@ -36,8 +35,7 @@ def test_export_import(device): if "EXPORT_IMPORT" not in core.get_property(device, "OPTIMIZATION_CAPABILITIES"): pytest.skip(f"{core.get_property(device, 'FULL_DEVICE_NAME')} plugin due-to export, import model API isn't implemented.") - model = core.read_model(model=test_net_xml, weights=test_net_bin) - compiled_model = core.compile_model(model, device) + compiled_model = generate_relu_compiled_model(device) user_stream = compiled_model.export_model() @@ -46,7 +44,7 @@ def test_export_import(device): img = generate_image() res = new_compiled.infer_new_request({"data": img}) - assert np.argmax(res[new_compiled.outputs[0]]) == 9 + assert np.argmax(res[new_compiled.outputs[0]]) == 531 def test_export_import_advanced(device): @@ -57,8 +55,7 @@ def test_export_import_advanced(device): if "EXPORT_IMPORT" not in core.get_property(device, "OPTIMIZATION_CAPABILITIES"): pytest.skip(f"{core.get_property(device, 'FULL_DEVICE_NAME')} plugin due-to export, import model API isn't implemented.") - model = core.read_model(model=test_net_xml, weights=test_net_bin) - compiled_model = core.compile_model(model, device) + compiled_model = generate_relu_compiled_model(device) user_stream = io.BytesIO() @@ -69,7 +66,7 @@ def test_export_import_advanced(device): img = generate_image() res = new_compiled.infer_new_request({"data": img}) - assert np.argmax(res[new_compiled.outputs[0]]) == 9 + assert np.argmax(res[new_compiled.outputs[0]]) == 531 @pytest.mark.parametrize("input_arguments", [[0], ["data"], []]) @@ -222,11 +219,15 @@ def test_direct_infer(device, shared_flag): assert np.array_equal(ref[compiled_model.outputs[0]], res[compiled_model.outputs[0]]) -def test_compiled_model_after_core_destroyed(device): +# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request +def test_compiled_model_after_core_destroyed(request, tmp_path, device): core = Core() - with open(test_net_bin, "rb") as f: + xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + model = get_relu_model() + serialize(model, xml_path, bin_path) + with open(bin_path, "rb") as f: weights = f.read() - with open(test_net_xml, "rb") as f: + with open(xml_path, "rb") as f: xml = f.read() model = core.read_model(model=xml, weights=weights) compiled = core.compile_model(model, device) diff --git a/src/bindings/python/tests/test_runtime/test_core.py b/src/bindings/python/tests/test_runtime/test_core.py index 86b8455f126..0bbcf684179 100644 --- a/src/bindings/python/tests/test_runtime/test_core.py +++ b/src/bindings/python/tests/test_runtime/test_core.py @@ -16,29 +16,21 @@ from openvino import ( CompiledModel, tensor_from_file, compile_model, + serialize, ) from openvino.runtime import Extension - -from tests.conftest import ( - model_path, - model_onnx_path, - get_model_with_template_extension, -) - -from tests.test_utils.test_utils import ( +from tests.utils.helpers import ( generate_image, generate_relu_compiled_model, get_relu_model, plugins_path, compare_models, + create_filename_for_test, + get_model_with_template_extension, ) -test_net_xml, test_net_bin = model_path() -test_net_onnx = model_onnx_path() - - def test_compact_api_xml(): img = generate_image() @@ -48,8 +40,12 @@ def test_compact_api_xml(): assert np.argmax(results[list(results)[0]]) == 531 -def test_compact_api_xml_posix_path(): - compiled_model = compile_model(Path(test_net_xml)) +# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request +def test_compact_api_xml_posix_path(request, tmp_path): + xml_path, _ = create_filename_for_test(request.node.name, tmp_path, True) + model = get_relu_model() + serialize(model, xml_path) + compiled_model = compile_model(Path(xml_path)) assert isinstance(compiled_model, CompiledModel) @@ -80,33 +76,51 @@ def test_core_class(device): assert np.allclose(results[list(results)[0]], expected_output) -def test_compile_model(device): +# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request +def test_compile_model(request, tmp_path, device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + relu_model = get_relu_model() + serialize(relu_model, xml_path, bin_path) + model = core.read_model(model=xml_path, weights=bin_path) compiled_model = core.compile_model(model, device) assert isinstance(compiled_model, CompiledModel) -def test_compile_model_without_device(): +# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request +def test_compile_model_without_device(request, tmp_path): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + relu_model = get_relu_model() + serialize(relu_model, xml_path, bin_path) + model = core.read_model(model=xml_path, weights=bin_path) compiled_model = core.compile_model(model) assert isinstance(compiled_model, CompiledModel) -def test_read_model_from_ir(): +# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request +def test_read_model_from_ir(request, tmp_path): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + relu_model = get_relu_model() + serialize(relu_model, xml_path, bin_path) + model = core.read_model(model=xml_path, weights=bin_path) assert isinstance(model, Model) - model = core.read_model(model=test_net_xml) + model = core.read_model(model=xml_path) assert isinstance(model, Model) -def test_read_model_from_tensor(): +# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request +def test_read_model_from_tensor(request, tmp_path): core = Core() - model = open(test_net_xml).read() - tensor = tensor_from_file(test_net_bin) + xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path, is_xml_path=True, is_bin_path=True) + relu_model = get_relu_model() + serialize(relu_model, xml_path, bin_path) + arr = np.ones(shape=(10), dtype=np.int8) + arr.tofile(bin_path) + model = open(xml_path).read() + tensor = tensor_from_file(bin_path) model = core.read_model(model=model, weights=tensor) assert isinstance(model, Model) @@ -118,48 +132,49 @@ def test_read_model_with_wrong_input(): assert "Provided python object type isn't supported as 'model' argument." in str(e.value) -def test_read_model_as_path(): +# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request +def test_read_model_as_path(request, tmp_path): core = Core() - model = core.read_model(model=Path(test_net_xml), weights=Path(test_net_bin)) + xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path, True, True) + relu_model = get_relu_model() + serialize(relu_model, xml_path, bin_path) + + model = core.read_model(model=Path(xml_path), weights=Path(bin_path)) assert isinstance(model, Model) - model = core.read_model(model=test_net_xml, weights=Path(test_net_bin)) + model = core.read_model(model=xml_path, weights=Path(bin_path)) assert isinstance(model, Model) - model = core.read_model(model=Path(test_net_xml)) + model = core.read_model(model=Path(xml_path)) assert isinstance(model, Model) -def test_read_model_from_onnx(): +# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request +def test_read_model_from_buffer(request, tmp_path): core = Core() - model = core.read_model(model=test_net_onnx) - assert isinstance(model, Model) - - -def test_read_model_from_onnx_as_path(): - core = Core() - model = core.read_model(model=Path(test_net_onnx)) - assert isinstance(model, Model) - - -def test_read_model_from_buffer(): - core = Core() - with open(test_net_bin, "rb") as f: + xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + relu_model = get_relu_model() + serialize(relu_model, xml_path, bin_path) + with open(bin_path, "rb") as f: weights = f.read() - with open(model_path()[0], "rb") as f: + with open(xml_path, "rb") as f: xml = f.read() model = core.read_model(model=xml, weights=weights) assert isinstance(model, Model) -def test_model_from_buffer_valid(): +# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request +def test_model_from_buffer_valid(request, tmp_path): core = Core() - with open(test_net_bin, "rb") as f: + xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + relu_model = get_relu_model() + serialize(relu_model, xml_path, bin_path) + with open(bin_path, "rb") as f: weights = f.read() - with open(model_path()[0], "rb") as f: + with open(xml_path, "rb") as f: xml = f.read() model = core.read_model(model=xml, weights=weights) - ref_model = core.read_model(model=test_net_xml, weights=test_net_bin) + ref_model = core.read_model(model=xml_path, weights=bin_path) assert compare_models(model, ref_model) @@ -255,7 +270,7 @@ def test_get_property_str(): def test_query_model(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() query_model = core.query_model(model=model, device_name=device) ops_model = model.get_ordered_ops() ops_func_names = [op.friendly_name for op in ops_model] @@ -326,7 +341,7 @@ def test_add_extension(): core = Core() core.add_extension(EmptyExtension()) core.add_extension([EmptyExtension(), EmptyExtension()]) - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() assert isinstance(model, Model) @@ -398,7 +413,7 @@ def test_read_model_from_buffer_no_weights(): def test_infer_new_request_return_type(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() img = generate_image() compiled_model = core.compile_model(model, device) res = compiled_model.infer_new_request({"data": img}) @@ -406,6 +421,6 @@ def test_infer_new_request_return_type(device): assert isinstance(arr, np.ndarray) assert arr.itemsize == 4 - assert arr.shape == (10,) + assert arr.shape == (3, 32, 32) assert arr.dtype == "float32" - assert arr.nbytes == 40 + assert arr.nbytes == 12288 diff --git a/src/bindings/python/tests/test_runtime/test_infer_request.py b/src/bindings/python/tests/test_runtime/test_infer_request.py index 3e2b20199bb..8fb76032831 100644 --- a/src/bindings/python/tests/test_runtime/test_infer_request.py +++ b/src/bindings/python/tests/test_runtime/test_infer_request.py @@ -27,10 +27,7 @@ from openvino.runtime import ProfilingInfo from openvino.preprocess import PrePostProcessor from tests import skip_need_mock_op -from tests.conftest import model_path -from tests.test_utils.test_utils import generate_image, get_relu_model - -test_net_xml, test_net_bin = model_path() +from tests.utils.helpers import generate_image, get_relu_model def create_model_with_memory(input_shape, data_type): @@ -100,7 +97,10 @@ def abs_model_with_data(device, ov_type, numpy_dtype): def test_get_profiling_info(device): core = Core() - model = core.read_model(test_net_xml, test_net_bin) + param = ops.parameter([1, 3, 32, 32], np.float32, name="data") + softmax = ops.softmax(param, 1, name="fc_out") + model = Model([softmax], [param], "test_model") + core.set_property(device, {"PERF_COUNT": "YES"}) compiled_model = core.compile_model(model, device) img = generate_image() @@ -164,22 +164,28 @@ def test_tensor_setter(device): def test_set_tensors(device): core = Core() - model = core.read_model(test_net_xml, test_net_bin) + + param = ops.parameter([1, 3, 32, 32], np.float32, name="data") + softmax = ops.softmax(param, 1, name="fc_out") + res = ops.result(softmax, name="res") + res.output(0).get_tensor().set_names({"res"}) + model = Model([res], [param], "test_model") + compiled_model = core.compile_model(model, device) data1 = generate_image() tensor1 = Tensor(data1) - data2 = np.ones(shape=(1, 10), dtype=np.float32) + data2 = np.ones(shape=(1, 3, 32, 32), dtype=np.float32) tensor2 = Tensor(data2) data3 = np.ones(shape=(1, 3, 32, 32), dtype=np.float32) tensor3 = Tensor(data3) - data4 = np.zeros(shape=(1, 10), dtype=np.float32) + data4 = np.zeros(shape=(1, 3, 32, 32), dtype=np.float32) tensor4 = Tensor(data4) request = compiled_model.create_infer_request() - request.set_tensors({"data": tensor1, "fc_out": tensor2}) + request.set_tensors({"data": tensor1, "res": tensor2}) t1 = request.get_tensor("data") - t2 = request.get_tensor("fc_out") + t2 = request.get_tensor("res") assert np.allclose(tensor1.data, t1.data, atol=1e-2, rtol=1e-2) assert np.allclose(tensor2.data, t2.data, atol=1e-2, rtol=1e-2) @@ -295,7 +301,7 @@ def test_inputs_outputs_property_and_method(device): @pytest.mark.skip(reason="Sporadically failed. Need further investigation. Ticket - 95967") def test_cancel(device): core = Core() - model = core.read_model(test_net_xml, test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) img = generate_image() request = compiled_model.create_infer_request() @@ -316,7 +322,7 @@ def test_cancel(device): @pytest.mark.parametrize("share_inputs", [True, False]) def test_start_async(device, share_inputs): core = Core() - model = core.read_model(test_net_xml, test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) img = generate_image() jobs = 3 @@ -482,7 +488,7 @@ def test_infer_queue(device, share_inputs): jobs = 8 num_request = 4 core = Core() - model = core.read_model(test_net_xml, test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) infer_queue = AsyncInferQueue(compiled_model, num_request) jobs_done = [{"finished": False, "latency": 0} for _ in range(jobs)] @@ -563,7 +569,7 @@ def test_infer_queue_fail_on_cpp_model(device): jobs = 6 num_request = 4 core = Core() - model = core.read_model(test_net_xml, test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) infer_queue = AsyncInferQueue(compiled_model, num_request) @@ -585,7 +591,7 @@ def test_infer_queue_fail_on_py_model(device): jobs = 1 num_request = 1 core = Core() - model = core.read_model(test_net_xml, test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) infer_queue = AsyncInferQueue(compiled_model, num_request) diff --git a/src/bindings/python/tests/test_runtime/test_input_node.py b/src/bindings/python/tests/test_runtime/test_input_node.py index 268f16fc006..6bdc421e9b5 100644 --- a/src/bindings/python/tests/test_runtime/test_input_node.py +++ b/src/bindings/python/tests/test_runtime/test_input_node.py @@ -4,19 +4,17 @@ import os -from ..conftest import model_path from openvino.runtime import Input, RTMap from openvino._pyopenvino import DescriptorTensor import openvino.runtime.opset12 as ops from openvino import Core, OVAny, Shape, PartialShape, Type - -test_net_xml, test_net_bin = model_path() +from tests.utils.helpers import get_relu_model def test_input_type(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) net_input = compiled_model.output(0) input_node = net_input.get_node().inputs()[0] @@ -25,7 +23,7 @@ def test_input_type(device): def test_const_output_docs(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) net_input = compiled_model.output(0) input_node = net_input.get_node().inputs()[0] @@ -35,7 +33,7 @@ def test_const_output_docs(device): def test_input_get_index(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) net_input = compiled_model.output(0) input_node = net_input.get_node().inputs()[0] @@ -44,7 +42,7 @@ def test_input_get_index(device): def test_input_element_type(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) net_input = compiled_model.output(0) input_node = net_input.get_node().inputs()[0] @@ -53,36 +51,36 @@ def test_input_element_type(device): def test_input_get_shape(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) net_input = compiled_model.output(0) input_node = net_input.get_node().inputs()[0] - assert str(input_node.get_shape()) == str(Shape([1, 10])) + assert str(input_node.get_shape()) == str(Shape([1, 3, 32, 32])) def test_input_get_partial_shape(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) net_input = compiled_model.output(0) input_node = net_input.get_node().inputs()[0] - expected_partial_shape = PartialShape([1, 10]) + expected_partial_shape = PartialShape([1, 3, 32, 32]) assert input_node.get_partial_shape() == expected_partial_shape def test_input_get_source_output(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) net_input = compiled_model.output(0) input_node = net_input.get_node().inputs()[0] name = input_node.get_source_output().get_node().get_friendly_name() - assert name == "fc_out" + assert name == "relu" def test_input_get_tensor(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) net_input = compiled_model.output(0) input_node = net_input.get_node().inputs()[0] @@ -92,7 +90,7 @@ def test_input_get_tensor(device): def test_input_get_rt_info(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) net_input = compiled_model.output(0) input_node = net_input.get_node().inputs()[0] @@ -102,7 +100,7 @@ def test_input_get_rt_info(device): def test_input_rt_info(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) net_input = compiled_model.output(0) input_node = net_input.get_node().inputs()[0] @@ -126,7 +124,7 @@ def test_input_replace_source_output(device): def test_input_update_rt_info(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) net_input = compiled_model.output(0) input_node = net_input.get_node().inputs()[0] diff --git a/src/bindings/python/tests/test_runtime/test_memory_modes.py b/src/bindings/python/tests/test_runtime/test_memory_modes.py index 41e4885a4b0..1fae8231bb4 100644 --- a/src/bindings/python/tests/test_runtime/test_memory_modes.py +++ b/src/bindings/python/tests/test_runtime/test_memory_modes.py @@ -9,7 +9,7 @@ import openvino.runtime as ov from openvino import Tensor from openvino.runtime.op import Constant -from tests.test_utils.test_utils import generate_image +from tests.utils.helpers import generate_image @pytest.mark.parametrize(("cls", "cls_str"), [ diff --git a/src/bindings/python/tests/test_runtime/test_model.py b/src/bindings/python/tests/test_runtime/test_model.py index 29c31d506bd..90e9ff4e4bb 100644 --- a/src/bindings/python/tests/test_runtime/test_model.py +++ b/src/bindings/python/tests/test_runtime/test_model.py @@ -23,7 +23,7 @@ from openvino import ( ) from openvino.runtime import Output -from tests.test_utils.test_utils import generate_add_model, create_filename_for_test +from tests.utils.helpers import generate_add_model, create_filename_for_test def test_test_descriptor_tensor(): diff --git a/src/bindings/python/tests/test_runtime/test_output_const_node.py b/src/bindings/python/tests/test_runtime/test_output_const_node.py index a4fc4cc7b72..5411658aef1 100644 --- a/src/bindings/python/tests/test_runtime/test_output_const_node.py +++ b/src/bindings/python/tests/test_runtime/test_output_const_node.py @@ -3,9 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 import os +import pytest from copy import copy, deepcopy -from ..conftest import model_path import openvino.runtime.opset12 as ops from openvino import ( Shape, @@ -19,16 +19,12 @@ from openvino.runtime import ( Output, RTMap, ) - -import pytest - - -test_net_xml, test_net_bin = model_path() +from tests.utils.helpers import get_relu_model def test_const_output_type(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) node = compiled_model.input(0) assert isinstance(node, ConstOutput) @@ -36,7 +32,7 @@ def test_const_output_type(device): def test_const_output_docs(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) node = compiled_model.input(0) exptected_string = "openvino.runtime.ConstOutput represents port/node output." @@ -45,7 +41,7 @@ def test_const_output_docs(device): def test_const_output_get_index(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) node = compiled_model.input("data") assert node.get_index() == 0 @@ -54,7 +50,7 @@ def test_const_output_get_index(device): def test_const_output_get_element_type(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) node = compiled_model.input("data") assert node.get_element_type() == Type.f32 @@ -63,7 +59,7 @@ def test_const_output_get_element_type(device): def test_const_output_get_shape(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) node = compiled_model.input("data") expected_shape = Shape([1, 3, 32, 32]) @@ -73,7 +69,7 @@ def test_const_output_get_shape(device): def test_const_output_get_partial_shape(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() exec_net = core.compile_model(model, device) node = exec_net.input("data") expected_partial_shape = PartialShape([1, 3, 32, 32]) @@ -83,7 +79,7 @@ def test_const_output_get_partial_shape(device): def test_const_output_get_target_inputs(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) outputs = compiled_model.outputs for node in outputs: @@ -93,7 +89,7 @@ def test_const_output_get_target_inputs(device): def test_const_output_get_names(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) input_name = "data" node = compiled_model.input(input_name) @@ -107,7 +103,7 @@ def test_const_output_get_names(device): def test_const_get_rf_info(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) output_node = compiled_model.output(0) rt_info = output_node.get_rt_info() @@ -116,7 +112,7 @@ def test_const_get_rf_info(device): def test_const_output_runtime_info(device): core = Core() - model = core.read_model(model=test_net_xml, weights=test_net_bin) + model = get_relu_model() compiled_model = core.compile_model(model, device) input_name = "data" output_node = compiled_model.input(input_name) diff --git a/src/bindings/python/tests/test_runtime/test_output_node.py b/src/bindings/python/tests/test_runtime/test_output_node.py index fe87e95199c..24faabc9f56 100644 --- a/src/bindings/python/tests/test_runtime/test_output_node.py +++ b/src/bindings/python/tests/test_runtime/test_output_node.py @@ -4,7 +4,6 @@ import os -from ..conftest import model_path import openvino.runtime.opset12 as ops from openvino import Type diff --git a/src/bindings/python/tests/test_runtime/test_properties.py b/src/bindings/python/tests/test_runtime/test_properties.py index f9ecbf9e8ee..1623e64e421 100644 --- a/src/bindings/python/tests/test_runtime/test_properties.py +++ b/src/bindings/python/tests/test_runtime/test_properties.py @@ -442,7 +442,7 @@ def test_properties_capability_gpu(): def test_properties_hint_model(): # Temporary imports - from tests.test_utils.test_utils import generate_add_model + from tests.utils.helpers import generate_add_model model = generate_add_model() diff --git a/src/bindings/python/tests/test_runtime/test_tensor.py b/src/bindings/python/tests/test_runtime/test_tensor.py index 9e6e1e547f0..46ebbb749fe 100644 --- a/src/bindings/python/tests/test_runtime/test_tensor.py +++ b/src/bindings/python/tests/test_runtime/test_tensor.py @@ -15,7 +15,7 @@ from openvino.helpers import pack_data, unpack_data import pytest -from tests.test_utils.test_utils import generate_image, generate_relu_compiled_model +from tests.utils.helpers import generate_image, generate_relu_compiled_model @pytest.mark.parametrize(("ov_type", "numpy_dtype"), [ diff --git a/src/bindings/python/tests/test_transformations/test_compression.py b/src/bindings/python/tests/test_transformations/test_compression.py index 85f74f6269b..f35b423b7df 100644 --- a/src/bindings/python/tests/test_transformations/test_compression.py +++ b/src/bindings/python/tests/test_transformations/test_compression.py @@ -9,7 +9,7 @@ from openvino.runtime.op import Parameter, Constant from openvino.runtime.opset12 import add, multiply import openvino as ov -from tests.test_utils.test_utils import create_filename_for_test +from tests.utils.helpers import create_filename_for_test def make_constant(values, transposed): diff --git a/src/bindings/python/tests/test_transformations/test_offline_api.py b/src/bindings/python/tests/test_transformations/test_offline_api.py index a1ba34d0e06..20543af89dc 100644 --- a/src/bindings/python/tests/test_transformations/test_offline_api.py +++ b/src/bindings/python/tests/test_transformations/test_offline_api.py @@ -19,7 +19,7 @@ from openvino._offline_transformations import ( from openvino.runtime import Model, PartialShape, Core, serialize, save_model import openvino.runtime as ov -from tests.test_utils.test_utils import create_filename_for_test, compare_models, _compare_models +from tests.utils.helpers import create_filename_for_test, compare_models, _compare_models def get_relu_model(): diff --git a/src/bindings/python/tests/test_transformations/test_public_transformations.py b/src/bindings/python/tests/test_transformations/test_public_transformations.py index b6b6f549f03..c030e5cb4b6 100644 --- a/src/bindings/python/tests/test_transformations/test_public_transformations.py +++ b/src/bindings/python/tests/test_transformations/test_public_transformations.py @@ -17,7 +17,7 @@ from openvino.runtime.passes import ( ) from tests.test_transformations.utils.utils import count_ops, get_relu_model -from tests.test_utils.test_utils import create_filename_for_test, compare_models +from tests.utils.helpers import create_filename_for_test, compare_models def get_model(): diff --git a/src/bindings/python/tests/test_utils/test_data_dispatch.py b/src/bindings/python/tests/test_utils/test_data_dispatch.py index e2ce00f10e7..8ee37c184fc 100644 --- a/src/bindings/python/tests/test_utils/test_data_dispatch.py +++ b/src/bindings/python/tests/test_utils/test_data_dispatch.py @@ -6,13 +6,11 @@ import os import pytest import numpy as np -from tests.conftest import model_path -from tests.test_utils.test_utils import generate_relu_compiled_model +from tests.utils.helpers import generate_relu_compiled_model from openvino.runtime import Model, ConstOutput, Type, Shape, Core, Tensor from openvino.runtime.utils.data_helpers import _data_dispatch is_myriad = os.environ.get("TEST_DEVICE") == "MYRIAD" -test_net_xml, test_net_bin = model_path(is_myriad) def _get_value(value): diff --git a/src/bindings/python/tests/test_utils/test_utils.py b/src/bindings/python/tests/test_utils/test_utils.py index e7c44340ebc..a7db0ff7d27 100644 --- a/src/bindings/python/tests/test_utils/test_utils.py +++ b/src/bindings/python/tests/test_utils/test_utils.py @@ -2,99 +2,19 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -from typing import Tuple, Union, List - -import os -import sys -import numpy as np import pytest - -from pathlib import Path - -import openvino -import openvino.runtime.opset12 as ops -from openvino.runtime import Model, Core, Shape from openvino.utils import deprecated +from tests.utils.helpers import compare_models, get_relu_model -def _compare_models(model_one: Model, model_two: Model, compare_names: bool = True) -> Tuple[bool, str]: # noqa: C901 the function is too complex - """Function to compare OpenVINO model (ops names, types and shapes). - - Note that the functions uses get_ordered_ops, so the topological order of ops should be also preserved. - - :param model_one: The first model to compare. - :param model_two: The second model to compare. - :param compare_names: Flag to control friendly names checking. Default: True - :return: Tuple which consists of bool value (True if models are equal, otherwise False) - and string with the message to reuse for debug/testing purposes. The string value - is empty when models are equal. - """ - result = True - msg = "" - - # Check friendly names of models - if compare_names and model_one.get_friendly_name() != model_two.get_friendly_name(): - result = False - msg += "Friendly names of models are not equal " - msg += f"model_one: {model_one.get_friendly_name()}, model_two: {model_two.get_friendly_name()}.\n" - - model_one_ops = model_one.get_ordered_ops() - model_two_ops = model_two.get_ordered_ops() - - # Check overall number of operators - if len(model_one_ops) != len(model_two_ops): - result = False - msg += "Not equal number of ops " - msg += f"model_one: {len(model_one_ops)}, model_two: {len(model_two_ops)}.\n" - - for i in range(len(model_one_ops)): - op_one_name = model_one_ops[i].get_friendly_name() # op from model_one - op_two_name = model_two_ops[i].get_friendly_name() # op from model_two - # Check friendly names - if (compare_names and op_one_name != op_two_name and model_one_ops[i].get_type_name() != "Constant"): - result = False - msg += "Not equal op names " - msg += f"model_one: {op_one_name}, " - msg += f"model_two: {op_two_name}.\n" - # Check output sizes - if model_one_ops[i].get_output_size() != model_two_ops[i].get_output_size(): - result = False - msg += f"Not equal output sizes of {op_one_name} and {op_two_name}.\n" - for idx in range(model_one_ops[i].get_output_size()): - # Check partial shapes of outputs - op_one_partial_shape = model_one_ops[i].get_output_partial_shape(idx) - op_two_partial_shape = model_two_ops[i].get_output_partial_shape(idx) - if op_one_partial_shape != op_two_partial_shape: - result = False - msg += f"Not equal op partial shapes of {op_one_name} and {op_two_name} on {idx} index " - msg += f"model_one: {op_one_partial_shape}, " - msg += f"model_two: {op_two_partial_shape}.\n" - # Check element types of outputs - op_one_element_type = model_one_ops[i].get_output_element_type(idx) - op_two_element_type = model_two_ops[i].get_output_element_type(idx) - if op_one_element_type != op_two_element_type: - result = False - msg += f"Not equal output element types of {op_one_name} and {op_two_name} on {idx} index " - msg += f"model_one: {op_one_element_type}, " - msg += f"model_two: {op_two_element_type}.\n" - - return result, msg - - -def compare_models(model_one: Model, model_two: Model, compare_names: bool = True): - """Function to compare OpenVINO model (ops names, types and shapes). - - :param model_one: The first model to compare. - :param model_two: The second model to compare. - :param compare_names: Flag to control friendly names checking. Default: True - :return: True if models are equal, otherwise raise an error with a report of mismatches. - """ - result, msg = _compare_models(model_one, model_two, compare_names=compare_names) - - if not result: - raise RuntimeError(msg) - - return result +def test_compare_functions(): + try: + from openvino.test_utils import compare_functions + model = get_relu_model() + status, _ = compare_functions(model, model) + assert status + except RuntimeError: + print("openvino.test_utils.compare_functions is not available") # noqa: T201 def test_compare_models_pass(): @@ -113,63 +33,6 @@ def test_compare_models_fail(): assert "Not equal op names model_one: data, model_two: ABC." in str(e.value) -def plugins_path(device, lib_path): - plugin_xml = f""" - - - - - """ - - with open("plugin_path.xml", "w") as f: - f.write(plugin_xml) - - plugins_paths = os.path.join(os.getcwd(), "plugin_path.xml") - return plugins_paths - - -def generate_image(shape: Tuple = (1, 3, 32, 32), dtype: Union[str, np.dtype] = "float32") -> np.array: - np.random.seed(42) - return np.random.rand(*shape).astype(dtype) - - -def get_relu_model(input_shape: List[int] = None, input_dtype=np.float32) -> openvino.runtime.Model: - if input_shape is None: - input_shape = [1, 3, 32, 32] - param = ops.parameter(input_shape, input_dtype, name="data") - relu = ops.relu(param, name="relu") - model = Model([relu], [param], "test_model") - model.get_ordered_ops()[2].friendly_name = "friendly" - - assert model is not None - return model - - -def generate_relu_compiled_model( - device, - input_shape: List[int] = None, - input_dtype=np.float32, -) -> openvino.runtime.CompiledModel: - if input_shape is None: - input_shape = [1, 3, 32, 32] - model = get_relu_model(input_shape, input_dtype) - core = Core() - return core.compile_model(model, device, {}) - - -def generate_model_and_image(device, input_shape: List[int] = None): - if input_shape is None: - input_shape = [1, 3, 32, 32] - return (generate_relu_compiled_model(device, input_shape), generate_image(input_shape)) - - -def generate_add_model() -> openvino._pyopenvino.Model: - param1 = ops.parameter(Shape([2, 1]), dtype=np.float32, name="data1") - param2 = ops.parameter(Shape([2, 1]), dtype=np.float32, name="data2") - add = ops.add(param1, param2) - return Model(add, [param1, param2], "TestFunction") - - def test_deprecation_decorator(): @deprecated() def deprecated_function1(param1, param2=None): @@ -195,21 +58,3 @@ def test_deprecation_decorator(): deprecated_function3() with pytest.warns(DeprecationWarning, match="deprecated_function4 is deprecated and will be removed in version 2025.4. Use another function instead"): deprecated_function4() - - -def create_filename_for_test(test_name, tmp_path, is_xml_path=False, is_bin_path=False): - """Return a tuple with automatically generated paths for xml and bin files. - - :param test_name: Name used in generating. - :param is_xml_path: True if xml file should be pathlib.Path object, otherwise return string. - :param is_bin_path: True if bin file should be pathlib.Path object, otherwise return string. - :return: Tuple with two objects representing xml and bin files. - """ - python_version = str(sys.version_info.major) + "_" + str(sys.version_info.minor) - filename = test_name.replace("test_", "").replace("[", "_").replace("]", "_") - filename = filename + "_" + python_version - path_to_xml = tmp_path / Path(filename + ".xml") - path_to_bin = tmp_path / Path(filename + ".bin") - _xml = path_to_xml if is_xml_path else str(path_to_xml) - _bin = path_to_bin if is_bin_path else str(path_to_bin) - return (_xml, _bin) diff --git a/src/bindings/python/tests/utils/helpers.py b/src/bindings/python/tests/utils/helpers.py new file mode 100644 index 00000000000..680e3ff41de --- /dev/null +++ b/src/bindings/python/tests/utils/helpers.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from typing import Tuple, Union, List + +import os +import sys +import numpy as np + +from sys import platform +from pathlib import Path + +import openvino +import openvino.runtime.opset12 as ops +from openvino.runtime import Model, Core, Shape + + +def _compare_models(model_one: Model, model_two: Model, compare_names: bool = True) -> Tuple[bool, str]: # noqa: C901 the function is too complex + """Function to compare OpenVINO model (ops names, types and shapes). + + Note that the functions uses get_ordered_ops, so the topological order of ops should be also preserved. + + :param model_one: The first model to compare. + :param model_two: The second model to compare. + :param compare_names: Flag to control friendly names checking. Default: True + :return: Tuple which consists of bool value (True if models are equal, otherwise False) + and string with the message to reuse for debug/testing purposes. The string value + is empty when models are equal. + """ + result = True + msg = "" + + # Check friendly names of models + if compare_names and model_one.get_friendly_name() != model_two.get_friendly_name(): + result = False + msg += "Friendly names of models are not equal " + msg += f"model_one: {model_one.get_friendly_name()}, model_two: {model_two.get_friendly_name()}.\n" + + model_one_ops = model_one.get_ordered_ops() + model_two_ops = model_two.get_ordered_ops() + + # Check overall number of operators + if len(model_one_ops) != len(model_two_ops): + result = False + msg += "Not equal number of ops " + msg += f"model_one: {len(model_one_ops)}, model_two: {len(model_two_ops)}.\n" + + for i in range(len(model_one_ops)): + op_one_name = model_one_ops[i].get_friendly_name() # op from model_one + op_two_name = model_two_ops[i].get_friendly_name() # op from model_two + # Check friendly names + if (compare_names and op_one_name != op_two_name and model_one_ops[i].get_type_name() != "Constant"): + result = False + msg += "Not equal op names " + msg += f"model_one: {op_one_name}, " + msg += f"model_two: {op_two_name}.\n" + # Check output sizes + if model_one_ops[i].get_output_size() != model_two_ops[i].get_output_size(): + result = False + msg += f"Not equal output sizes of {op_one_name} and {op_two_name}.\n" + for idx in range(model_one_ops[i].get_output_size()): + # Check partial shapes of outputs + op_one_partial_shape = model_one_ops[i].get_output_partial_shape(idx) + op_two_partial_shape = model_two_ops[i].get_output_partial_shape(idx) + if op_one_partial_shape != op_two_partial_shape: + result = False + msg += f"Not equal op partial shapes of {op_one_name} and {op_two_name} on {idx} index " + msg += f"model_one: {op_one_partial_shape}, " + msg += f"model_two: {op_two_partial_shape}.\n" + # Check element types of outputs + op_one_element_type = model_one_ops[i].get_output_element_type(idx) + op_two_element_type = model_two_ops[i].get_output_element_type(idx) + if op_one_element_type != op_two_element_type: + result = False + msg += f"Not equal output element types of {op_one_name} and {op_two_name} on {idx} index " + msg += f"model_one: {op_one_element_type}, " + msg += f"model_two: {op_two_element_type}.\n" + + return result, msg + + +def compare_models(model_one: Model, model_two: Model, compare_names: bool = True): + """Function to compare OpenVINO model (ops names, types and shapes). + + :param model_one: The first model to compare. + :param model_two: The second model to compare. + :param compare_names: Flag to control friendly names checking. Default: True + :return: True if models are equal, otherwise raise an error with a report of mismatches. + """ + result, msg = _compare_models(model_one, model_two, compare_names=compare_names) + + if not result: + raise RuntimeError(msg) + + return result + + +def plugins_path(device, lib_path): + plugin_xml = f""" + + + + + """ + + with open("plugin_path.xml", "w") as f: + f.write(plugin_xml) + + plugins_paths = os.path.join(os.getcwd(), "plugin_path.xml") + return plugins_paths + + +def generate_image(shape: Tuple = (1, 3, 32, 32), dtype: Union[str, np.dtype] = "float32") -> np.array: + np.random.seed(42) + return np.random.rand(*shape).astype(dtype) + + +def get_model_with_template_extension(): + core = Core() + ir = bytes(b""" + + + + + + 1 + 3 + 22 + 22 + + + + + + + 1 + 3 + 22 + 22 + + + + + 1 + 3 + 22 + 22 + + + + + + + 1 + 3 + 22 + 22 + + + + + + + + +""") + if platform == "win32": + core.add_extension(library_path="openvino_template_extension.dll") + else: + core.add_extension(library_path="libopenvino_template_extension.so") + return core, core.read_model(ir) + + +def get_relu_model(input_shape: List[int] = None, input_dtype=np.float32) -> openvino.runtime.Model: + if input_shape is None: + input_shape = [1, 3, 32, 32] + param = ops.parameter(input_shape, input_dtype, name="data") + relu = ops.relu(param, name="relu") + model = Model([relu], [param], "test_model") + model.get_ordered_ops()[2].friendly_name = "friendly" + + assert model is not None + return model + + +def generate_relu_compiled_model( + device, + input_shape: List[int] = None, + input_dtype=np.float32, +) -> openvino.runtime.CompiledModel: + if input_shape is None: + input_shape = [1, 3, 32, 32] + model = get_relu_model(input_shape, input_dtype) + core = Core() + return core.compile_model(model, device, {}) + + +def generate_model_and_image(device, input_shape: List[int] = None): + if input_shape is None: + input_shape = [1, 3, 32, 32] + return (generate_relu_compiled_model(device, input_shape), generate_image(input_shape)) + + +def generate_add_model() -> openvino._pyopenvino.Model: + param1 = ops.parameter(Shape([2, 1]), dtype=np.float32, name="data1") + param2 = ops.parameter(Shape([2, 1]), dtype=np.float32, name="data2") + add = ops.add(param1, param2) + return Model(add, [param1, param2], "TestFunction") + + +def create_filename_for_test(test_name, tmp_path, is_xml_path=False, is_bin_path=False): + """Return a tuple with automatically generated paths for xml and bin files. + + :param test_name: Name used in generating. + :param is_xml_path: True if xml file should be pathlib.Path object, otherwise return string. + :param is_bin_path: True if bin file should be pathlib.Path object, otherwise return string. + :return: Tuple with two objects representing xml and bin files. + """ + python_version = str(sys.version_info.major) + "_" + str(sys.version_info.minor) + filename = test_name.replace("test_", "").replace("[", "_").replace("]", "_") + filename = filename + "_" + python_version + path_to_xml = tmp_path / Path(filename + ".xml") + path_to_bin = tmp_path / Path(filename + ".bin") + _xml = path_to_xml if is_xml_path else str(path_to_xml) + _bin = path_to_bin if is_bin_path else str(path_to_bin) + return (_xml, _bin) diff --git a/src/bindings/python/tests/test_utils/utils/test_model.onnx b/src/bindings/python/tests/utils/utils/test_model.onnx similarity index 100% rename from src/bindings/python/tests/test_utils/utils/test_model.onnx rename to src/bindings/python/tests/utils/utils/test_model.onnx diff --git a/src/bindings/python/tests/test_utils/utils/test_model_fp16.bin b/src/bindings/python/tests/utils/utils/test_model_fp16.bin similarity index 100% rename from src/bindings/python/tests/test_utils/utils/test_model_fp16.bin rename to src/bindings/python/tests/utils/utils/test_model_fp16.bin diff --git a/src/bindings/python/tests/test_utils/utils/test_model_fp16.xml b/src/bindings/python/tests/utils/utils/test_model_fp16.xml similarity index 100% rename from src/bindings/python/tests/test_utils/utils/test_model_fp16.xml rename to src/bindings/python/tests/utils/utils/test_model_fp16.xml diff --git a/src/bindings/python/tests/test_utils/utils/test_model_fp32.bin b/src/bindings/python/tests/utils/utils/test_model_fp32.bin similarity index 100% rename from src/bindings/python/tests/test_utils/utils/test_model_fp32.bin rename to src/bindings/python/tests/utils/utils/test_model_fp32.bin diff --git a/src/bindings/python/tests/test_utils/utils/test_model_fp32.xml b/src/bindings/python/tests/utils/utils/test_model_fp32.xml similarity index 100% rename from src/bindings/python/tests/test_utils/utils/test_model_fp32.xml rename to src/bindings/python/tests/utils/utils/test_model_fp32.xml