diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c43e6edf501..3f2178a1c68 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -93,6 +93,7 @@ /tests/layer_tests/pytorch_tests/ @openvinotoolkit/openvino-pytorch-frontend-maintainers /tests/layer_tests/tensorflow_tests @openvinotoolkit/openvino-tf-frontend-maintainers /tests/layer_tests/jax_tests @openvinotoolkit/openvino-tf-frontend-maintainers +/tests/model_hub_tests @openvinotoolkit/openvino-tf-frontend-maintainers # Tools: /tools/ @openvinotoolkit/openvino-tools-maintainers diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 5a0e28a8a31..7fe5e42e216 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -41,6 +41,7 @@ jobs: INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests SAMPLES_INSTALL_DIR: ${{ github.workspace }}/install/samples LAYER_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/layer_tests + MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests BUILD_DIR: ${{ github.workspace }}/build DATA_PATH: ${{ github.workspace }}/testdata MODELS_PATH: ${{ github.workspace }}/testdata @@ -183,15 +184,24 @@ jobs: - name: Cmake Layer Tests run: cmake -GNinja -S ${{ env.OPENVINO_REPO }}/tests/layer_tests -B ${{ env.BUILD_DIR }}/layer_tests + - name: Cmake Model Hub Tests + run: cmake -GNinja -S ${{ env.OPENVINO_REPO }}/tests/model_hub_tests -B ${{ env.BUILD_DIR }}/model_hub_tests + - name: Build Layer Tests run: cmake --build ${{ env.BUILD_DIR }}/layer_tests --parallel --config Release + - name: Build Model Hub Tests + run: cmake --build ${{ env.BUILD_DIR }}/model_hub_tests --parallel --config Release + - name: Install wheel packages run: cmake -DCOMPONENT=python_wheels -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -P ${{ env.BUILD_DIR }}/cmake_install.cmake - name: Install Layer Tests run: cmake -DCOMPONENT=tests -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -P ${{ env.BUILD_DIR }}/layer_tests/cmake_install.cmake + - name: Install Model Hub Tests + run: cmake -DCOMPONENT=tests -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -P ${{ env.BUILD_DIR }}/model_hub_tests/cmake_install.cmake + - name: Install python wheels run: python3 -m pip install openvino-dev --find-links=${{ env.INSTALL_DIR }}/tools @@ -482,6 +492,7 @@ jobs: INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests SAMPLES_INSTALL_DIR: ${{ github.workspace }}/install/samples LAYER_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/layer_tests + MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests BUILD_DIR: ${{ github.workspace }}/build DATA_PATH: ${{ github.workspace }}/testdata MODELS_PATH: ${{ github.workspace }}/testdata @@ -655,6 +666,18 @@ jobs: env: TEST_DEVICE: CPU + - name: TensorFlow Hub Tests - TF FE + run: | + python3 -m pip install -r ${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}/tf_hub_tests/requirements.txt + + export PYTHONPATH=${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}:$PYTHONPATH + + source ${{ env.INSTALL_DIR }}/setupvars.sh + + python3 -m pytest ${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}/tf_hub_tests/ -m precommit --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf_hub_tf_fe.xml + env: + TEST_DEVICE: CPU + - name: TensorFlow 1 Layer Tests - Legacy FE run: | python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt diff --git a/tests/model_hub_tests/CMakeLists.txt b/tests/model_hub_tests/CMakeLists.txt new file mode 100644 index 00000000000..58bc74c76ef --- /dev/null +++ b/tests/model_hub_tests/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +cmake_minimum_required(VERSION 3.13) + +project(model_hub_tests) + +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL) diff --git a/tests/model_hub_tests/models_hub_common/constants.py b/tests/model_hub_tests/models_hub_common/constants.py new file mode 100644 index 00000000000..65abfdf436a --- /dev/null +++ b/tests/model_hub_tests/models_hub_common/constants.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os + +# supported_devices : CPU, GPU, GNA +test_device = os.environ.get('TEST_DEVICE', 'CPU;GPU').split(';') diff --git a/tests/model_hub_tests/models_hub_common/test_convert_model.py b/tests/model_hub_tests/models_hub_common/test_convert_model.py new file mode 100644 index 00000000000..4398f7e4a95 --- /dev/null +++ b/tests/model_hub_tests/models_hub_common/test_convert_model.py @@ -0,0 +1,86 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import numpy as np +from openvino.runtime import Core +from openvino.tools.mo import convert_model + + +class TestConvertModel: + def load_model(self, model_name, model_link): + raise "load_model is not implemented" + + def get_inputs_info(self, model_obj): + raise "get_inputs_info is not implemented" + + def prepare_input(self, input_shape, input_type): + if input_type in [np.float32, np.float64]: + return np.random.randint(-2, 2, size=input_shape).astype(input_type) + elif input_type in [np.int8, np.int16, np.int32, np.int64]: + return np.random.randint(-5, 5, size=input_shape).astype(input_type) + elif input_type in [np.uint8, np.uint16]: + return np.random.randint(0, 5, size=input_shape).astype(input_type) + elif input_type in [str]: + return np.broadcast_to("Some string", input_shape) + elif input_type in [bool]: + return np.random.randint(0, 2, size=input_shape).astype(input_type) + else: + assert False, "Unsupported type {}".format(input_type) + + def prepare_inputs(self, inputs_info): + inputs = [] + for input_shape, input_type in inputs_info: + inputs.append(self.prepare_input(input_shape, input_type)) + return inputs + + def convert_model(self, model_obj): + ov_model = convert_model(model_obj) + return ov_model + + def infer_fw_model(self, model_obj, inputs): + raise "infer_fw_model is not implemented" + + def infer_ov_model(self, ov_model, inputs, ie_device): + core = Core() + compiled = core.compile_model(ov_model, ie_device) + ov_outputs = compiled(inputs) + return ov_outputs + + def compare_results(self, fw_outputs, ov_outputs): + assert len(fw_outputs) == len(ov_outputs), \ + "Different number of outputs between TensorFlow and OpenVINO:" \ + " {} vs. {}".format(len(fw_outputs), len(ov_outputs)) + + fw_eps = 5e-2 + is_ok = True + for out_name in fw_outputs.keys(): + cur_fw_res = fw_outputs[out_name] + assert out_name in ov_outputs, \ + "OpenVINO outputs does not contain tensor with name {}".format(out_name) + cur_ov_res = ov_outputs[out_name] + print(f"fw_re: {cur_fw_res};\n ov_res: {cur_ov_res}") + if not np.allclose(cur_ov_res, cur_fw_res, + atol=fw_eps, + rtol=fw_eps, equal_nan=True): + is_ok = False + print("Max diff is {}".format(np.array(abs(cur_ov_res - cur_fw_res)).max())) + else: + print("Accuracy validation successful!\n") + print("absolute eps: {}, relative eps: {}".format(fw_eps, fw_eps)) + assert is_ok, "Accuracy validation failed" + + def run(self, model_name, model_link, ie_device): + print("Load the model {} (url: {})".format(model_name, model_link)) + concrete_func = self.load_model(model_name, model_link) + print("Retrieve inputs info") + inputs_info = self.get_inputs_info(concrete_func) + print("Prepare input data") + inputs = self.prepare_inputs(inputs_info) + print("Convert the model into ov::Model") + ov_model = self.convert_model(concrete_func) + print("Infer the original model") + fw_outputs = self.infer_fw_model(concrete_func, inputs) + print("Infer ov::Model") + ov_outputs = self.infer_ov_model(ov_model, inputs, ie_device) + print("Compare TensorFlow and OpenVINO results") + self.compare_results(fw_outputs, ov_outputs) diff --git a/tests/model_hub_tests/models_hub_common/utils.py b/tests/model_hub_tests/models_hub_common/utils.py new file mode 100644 index 00000000000..b2b31c72707 --- /dev/null +++ b/tests/model_hub_tests/models_hub_common/utils.py @@ -0,0 +1,24 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import itertools + +from models_hub_common.constants import test_device + + +def get_models_list(file_name: str): + models = [] + with open(file_name) as f: + for model_info in f: + model_name, model_link = model_info.split(',') + models.append((model_name, model_link)) + return models + + +def get_params(ie_device=None): + ie_device_params = ie_device if ie_device else test_device + + test_args = [] + for element in itertools.product(ie_device_params): + test_args.append(element) + return test_args diff --git a/tests/model_hub_tests/tf_hub_tests/conftest.py b/tests/model_hub_tests/tf_hub_tests/conftest.py new file mode 100644 index 00000000000..3f39a5be8c2 --- /dev/null +++ b/tests/model_hub_tests/tf_hub_tests/conftest.py @@ -0,0 +1,12 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import inspect + +from models_hub_common.utils import get_params + + +def pytest_generate_tests(metafunc): + test_gen_attrs_names = list(inspect.signature(get_params).parameters) + params = get_params() + metafunc.parametrize(test_gen_attrs_names, params, scope="function") diff --git a/tests/model_hub_tests/tf_hub_tests/precommit_models b/tests/model_hub_tests/tf_hub_tests/precommit_models new file mode 100644 index 00000000000..3475587a538 --- /dev/null +++ b/tests/model_hub_tests/tf_hub_tests/precommit_models @@ -0,0 +1 @@ +vision/embedder/fungi_V2,https://tfhub.dev/svampeatlas/vision/embedder/fungi_V2/1?tf-hub-format=compressed \ No newline at end of file diff --git a/tests/model_hub_tests/tf_hub_tests/requirements.txt b/tests/model_hub_tests/tf_hub_tests/requirements.txt new file mode 100644 index 00000000000..6c80a226426 --- /dev/null +++ b/tests/model_hub_tests/tf_hub_tests/requirements.txt @@ -0,0 +1,5 @@ +-c ../../constraints.txt +numpy +pytest +tensorflow +tensorflow-hub diff --git a/tests/model_hub_tests/tf_hub_tests/test_tf_hub_convert_model.py b/tests/model_hub_tests/tf_hub_tests/test_tf_hub_convert_model.py new file mode 100644 index 00000000000..98deed4b0eb --- /dev/null +++ b/tests/model_hub_tests/tf_hub_tests/test_tf_hub_convert_model.py @@ -0,0 +1,74 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os + +import numpy as np +import pytest +import tensorflow as tf +import tensorflow_hub as hub +from models_hub_common.test_convert_model import TestConvertModel +from models_hub_common.utils import get_models_list + + +class TestTFHubConvertModel(TestConvertModel): + def load_model(self, model_name, model_link): + load = hub.load(model_link) + if 'default' in list(load.signatures.keys()): + concrete_func = load.signatures['default'] + else: + signature_keys = sorted(list(load.signatures.keys())) + assert len(signature_keys) > 0, "No signatures for a model {}, url {}".format(model_name, model_link) + concrete_func = load.signatures[signature_keys[0]] + return concrete_func + + def get_inputs_info(self, model_obj): + inputs_info = [] + for input_info in model_obj.inputs: + input_shape = [] + for dim in input_info.shape.as_list(): + if dim is None: + input_shape.append(1) + else: + input_shape.append(dim) + type_map = { + tf.float64: np.float64, + tf.float32: np.float32, + tf.int8: np.int8, + tf.int16: np.int16, + tf.int32: np.int32, + tf.int64: np.int64, + tf.uint8: np.uint8, + tf.uint16: np.uint16, + tf.string: str, + tf.bool: bool, + } + assert input_info.dtype in type_map, "Unsupported input type: {}".format(input_info.dtype) + inputs_info.append((input_shape, type_map[input_info.dtype])) + + return inputs_info + + def infer_fw_model(self, model_obj, inputs): + tf_inputs = [] + for input_data in inputs: + tf_inputs.append(tf.constant(input_data)) + output_dict = {} + for out_name, out_value in model_obj(*tf_inputs).items(): + output_dict[out_name] = out_value.numpy() + + # map external tensor names to internal names + # TODO: remove this workaround + fw_outputs = {} + for out_name, out_value in output_dict.items(): + mapped_name = out_name + if out_name in model_obj.structured_outputs: + mapped_name = model_obj.structured_outputs[out_name].name + fw_outputs[mapped_name] = out_value + return fw_outputs + + @pytest.mark.parametrize("model_name,model_link", + get_models_list(os.path.join(os.path.dirname(__file__), "precommit_models"))) + @pytest.mark.precommit + def test_convert_model(self, model_name, model_link, ie_device): + # we do not perform transpose in the test in case of new frontend + self.run(model_name, model_link, ie_device)