[PT FE] Add tests for torchvision models (#19702)

* [PT FE] Add tests for torchvision models

* Update tests/model_hub_tests/torch_tests/requirements.txt

* Apply review comments

* Clean tmp directory and make this separate job

* Update .github/workflows/linux.yml

* Update tests/model_hub_tests/torch_tests/test_torchvision_models.py

.github/workflows/linux.yml

@@ -867,3 +867,63 @@ jobs:
           path: |
             ${{ env.INSTALL_TEST_DIR }}/TEST*.html
           if-no-files-found: 'error'
+
+  PyTorch_Models_Tests:
+    needs: Build
+    defaults:
+      run:
+        shell: bash
+    runs-on: ubuntu-22.04
+    env:
+      INSTALL_DIR: ${{ github.workspace }}/install
+      INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
+      MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests
+
+    steps:
+      - name: Create Directories
+        run: |
+          mkdir -p ${{ env.INSTALL_DIR }} ${{ env.INSTALL_TEST_DIR }}
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Download OpenVINO package
+        uses: actions/download-artifact@v3
+        with:
+          name: openvino_package
+          path: ${{ env.INSTALL_DIR }}
+
+      - name: Download OpenVINO tests package
+        uses: actions/download-artifact@v3
+        with:
+          name: openvino_tests
+          path: ${{ env.INSTALL_TEST_DIR }}
+
+      - name: Extract OpenVINO packages
+        run: |
+          pushd ${{ env.INSTALL_DIR }}
+            tar -xzf openvino_package.tar.gz -C ${{ env.INSTALL_DIR }} && rm openvino_package.tar.gz || exit 1
+          popd
+          pushd ${{ env.INSTALL_TEST_DIR }}
+            tar -xzf openvino_tests.tar.gz -C ${{ env.INSTALL_DIR }} && rm openvino_tests.tar.gz || exit 1
+          popd
+      - name: Install Python wheels
+        run: |
+          python3 -m pip install openvino --find-links=${{ env.INSTALL_DIR }}/tools
+      - name: PyTorch Models Tests
+        run: |
+          python3 -m pip install -r ${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}/torch_tests/requirements.txt
+          export PYTHONPATH=${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}:$PYTHONPATH
+          python3 -m pytest ${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}/torch_tests/ -m ${{ env.TYPE }} --html=${{ env.INSTALL_TEST_DIR }}/TEST-torch_model_tests.html --self-contained-html
+        env:
+          TYPE: ${{ github.event_name == 'schedule' && 'nightly' || 'precommit'}}
+          TEST_DEVICE: CPU
+
+      - name: Upload Test Results
+        uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: test-results-torch-models
+          path: |
+            ${{ env.INSTALL_TEST_DIR }}/TEST*.html
+          if-no-files-found: 'error'

tests/model_hub_tests/models_hub_common/test_convert_model.py

@@ -4,6 +4,7 @@ import gc
 
 import numpy as np
 from models_hub_common.multiprocessing_utils import multiprocessing_run
+from models_hub_common.utils import compare_two_tensors
 from openvino import convert_model
 from openvino.runtime import Core
 
@@ -59,20 +60,20 @@ class TestConvertModel:
 
         fw_eps = 5e-2
         is_ok = True
-        for out_name in fw_outputs.keys():
-            cur_fw_res = fw_outputs[out_name]
-            assert out_name in ov_outputs, \
-                "OpenVINO outputs does not contain tensor with name {}".format(out_name)
-            cur_ov_res = ov_outputs[out_name]
-            print(f"fw_re: {cur_fw_res};\n ov_res: {cur_ov_res}")
-            if not np.allclose(cur_ov_res, cur_fw_res,
-                               atol=fw_eps,
-                               rtol=fw_eps, equal_nan=True):
-                is_ok = False
-                print("Max diff is {}".format(np.array(abs(cur_ov_res - cur_fw_res)).max()))
-            else:
-                print("Accuracy validation successful!\n")
-                print("absolute eps: {}, relative eps: {}".format(fw_eps, fw_eps))
+        if isinstance(fw_outputs, dict):
+            for out_name in fw_outputs.keys():
+                cur_fw_res = fw_outputs[out_name]
+                assert out_name in ov_outputs, \
+                    "OpenVINO outputs does not contain tensor with name {}".format(out_name)
+                cur_ov_res = ov_outputs[out_name]
+                print(f"fw_re: {cur_fw_res};\n ov_res: {cur_ov_res}")
+                is_ok = compare_two_tensors(cur_ov_res, cur_fw_res, fw_eps)
+        else:
+            for i in range(len(ov_outputs)):
+                cur_fw_res = fw_outputs[i]
+                cur_ov_res = ov_outputs[i]
+                print(f"fw_re: {cur_fw_res};\n ov_res: {cur_ov_res}")
+                is_ok = compare_two_tensors(cur_ov_res, cur_fw_res, fw_eps)
         assert is_ok, "Accuracy validation failed"
 
     def teardown_method(self):

tests/model_hub_tests/models_hub_common/utils.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import itertools
+import numpy as np
 
 from models_hub_common.constants import test_device
 
@@ -27,6 +28,17 @@ def get_models_list(file_name: str):
     return models
 
 
+def compare_two_tensors(ov_res, fw_res, eps):
+    is_ok = True
+    if not np.allclose(ov_res, fw_res, atol=eps, rtol=eps, equal_nan=True):
+        is_ok = False
+        print("Max diff is {}".format(np.array(abs(ov_res - fw_res)).max()))
+    else:
+        print("Accuracy validation successful!\n")
+        print("absolute eps: {}, relative eps: {}".format(eps, eps))
+    return is_ok
+
+
 def get_params(ie_device=None):
     ie_device_params = ie_device if ie_device else test_device
 

tests/model_hub_tests/torch_tests/conftest.py

@@ -0,0 +1,12 @@
+# Copyright (C) 2018-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import inspect
+
+from models_hub_common.utils import get_params
+
+
+def pytest_generate_tests(metafunc):
+    test_gen_attrs_names = list(inspect.signature(get_params).parameters)
+    params = get_params()
+    metafunc.parametrize(test_gen_attrs_names, params, scope="function")

tests/model_hub_tests/torch_tests/requirements.txt

@@ -0,0 +1,7 @@
+-c ../../constraints.txt
+numpy
+pytest
+pytest-html
+torch
+torchvision
+av

tests/model_hub_tests/torch_tests/test_torchvision_models.py

@@ -0,0 +1,115 @@
+# Copyright (C) 2018-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+import torch
+import tempfile
+import torchvision.transforms.functional as F
+from models_hub_common.test_convert_model import TestConvertModel
+from openvino import convert_model
+
+
+def get_all_models() -> list:
+    m_list = torch.hub.list("pytorch/vision")
+    m_list.remove("get_model_weights")
+    m_list.remove("get_weight")
+    return m_list
+
+
+def get_video():
+    """
+    Download video and return frames.
+    Using free video from pexels.com, credits go to Pavel Danilyuk.
+    Initially used in https://pytorch.org/vision/stable/auto_examples/plot_optical_flow.html
+    """
+    from pathlib import Path
+    from urllib.request import urlretrieve
+    from torchvision.io import read_video
+
+    video_url = "https://download.pytorch.org/tutorial/pexelscom_pavel_danilyuk_basketball_hd.mp4"
+    with tempfile.TemporaryDirectory() as tmp:
+        video_path = Path(tmp) / "basketball.mp4"
+        _ = urlretrieve(video_url, video_path)
+
+        frames, _, _ = read_video(str(video_path), output_format="TCHW")
+    return frames
+
+
+def prepare_frames_for_raft(name, frames1, frames2):
+    w = torch.hub.load("pytorch/vision", "get_model_weights", name=name).DEFAULT
+    img1_batch = torch.stack(frames1)
+    img2_batch = torch.stack(frames2)
+    img1_batch = F.resize(img1_batch, size=[520, 960], antialias=False)
+    img2_batch = F.resize(img2_batch, size=[520, 960], antialias=False)
+    img1_batch, img2_batch = w.transforms()(img1_batch, img2_batch)
+    return (img1_batch, img2_batch)
+
+
+# To make tests reproducible we seed the random generator
+torch.manual_seed(0)
+
+
+class TestTorchHubConvertModel(TestConvertModel):
+    def setup_method(self):
+        self.cache_dir = tempfile.TemporaryDirectory()
+        # set temp dir for torch cache
+        torch.hub.set_dir(str(self.cache_dir.name))
+
+    def load_model(self, model_name, model_link):
+        m = torch.hub.load("pytorch/vision", model_name, weights='DEFAULT')
+        m.eval()
+        if model_name == "s3d" or any([m in model_name for m in ["swin3d", "r3d_18", "mc3_18", "r2plus1d_18"]]):
+            self.example = (torch.randn([1, 3, 224, 224, 224]),)
+            self.inputs = (torch.randn([1, 3, 224, 224, 224]),)
+        elif "mvit" in model_name:
+            # 16 frames from video
+            self.example = (torch.randn(1, 3, 16, 224, 224),)
+            self.inputs = (torch.randn(1, 3, 16, 224, 224),)
+        elif "raft" in model_name:
+            frames = get_video()
+            self.example = prepare_frames_for_raft(model_name,
+                                                   [frames[100], frames[150]],
+                                                   [frames[101], frames[151]])
+            self.inputs = prepare_frames_for_raft(model_name,
+                                                  [frames[75], frames[125]],
+                                                  [frames[76], frames[126]])
+        else:
+            self.example = (torch.randn(1, 3, 224, 224),)
+            self.inputs = (torch.randn(1, 3, 224, 224),)
+        return m
+
+    def get_inputs_info(self, model_obj):
+        return None
+
+    def prepare_inputs(self, inputs_info):
+        return [i.numpy() for i in self.inputs]
+
+    def convert_model(self, model_obj):
+        ov_model = convert_model(model_obj, example_input=self.example)
+        return ov_model
+
+    def infer_fw_model(self, model_obj, inputs):
+        fw_outputs = model_obj(*[torch.from_numpy(i) for i in inputs])
+        if isinstance(fw_outputs, dict):
+            for k in fw_outputs.keys():
+                fw_outputs[k] = fw_outputs[k].numpy(force=True)
+        elif isinstance(fw_outputs, (list, tuple)):
+            fw_outputs = [o.numpy(force=True) for o in fw_outputs]
+        else:
+            fw_outputs = [fw_outputs.numpy(force=True)]
+        return fw_outputs
+
+    def teardown_method(self):
+        # cleanup tmpdir
+        self.cache_dir.cleanup()
+        super().teardown_method()
+
+    @pytest.mark.parametrize("model_name", ["efficientnet_b7", "raft_small", "swin_v2_s"])
+    @pytest.mark.precommit
+    def test_convert_model_precommit(self, model_name, ie_device):
+        self.run(model_name, None, ie_device)
+
+    @pytest.mark.parametrize("model_name", get_all_models())
+    @pytest.mark.nightly
+    def test_convert_model_all_models(self, model_name, ie_device):
+        self.run(model_name, None, ie_device)