[PT FE] Add tests for timm models (#19997)

* Add tests for timm models

* Apply suggestions from code review

* Some improvements

* Update tests/model_hub_tests/torch_tests/test_transformers.py

tests/model_hub_tests/models_hub_common/constants.py

@@ -11,5 +11,11 @@ else:
     tf_hub_cache_dir = os.path.join(tempfile.gettempdir(), "tfhub_modules")
     os.environ['TFHUB_CACHE_DIR'] = tf_hub_cache_dir
 
+if os.environ.get('HUGGINGFACE_HUB_CACHE') is not None:
+    hf_hub_cache_dir = os.environ['HUGGINGFACE_HUB_CACHE']
+else:
+    hf_hub_cache_dir = tempfile.gettempdir()
+    os.environ['HUGGINGFACE_HUB_CACHE'] = hf_hub_cache_dir
+
 # supported_devices : CPU, GPU, GNA
 test_device = os.environ.get('TEST_DEVICE', 'CPU;GPU').split(';')

tests/model_hub_tests/models_hub_common/utils.py

@@ -1,6 +1,8 @@
 # Copyright (C) 2018-2023 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+import os
+import shutil
 import itertools
 import numpy as np
 
@@ -47,3 +49,16 @@ def get_params(ie_device=None):
     for element in itertools.product(ie_device_params):
         test_args.append(element)
     return test_args
+
+
+def cleanup_dir(dir: str):
+    # remove all downloaded files from cache
+    for file_name in os.listdir(dir):
+        file_path = os.path.join(dir, file_name)
+        try:
+            if os.path.isfile(file_path) or os.path.islink(file_path):
+                os.unlink(file_path)
+            elif os.path.isdir(file_path):
+                shutil.rmtree(file_path)
+        except Exception as e:
+            pass

tests/model_hub_tests/torch_tests/hf_transformers_models

@@ -3,7 +3,7 @@ abcp4/mymodel-test,mymodel,skip,Load problem
 abeja/gpt-neox-japanese-2.7b,gpt_neox_japanese
 acl-submission-anonym/EAM-spectral,examuse,skip,Load problem
 adalbertojunior/modular-test,modular,skip,Load problem
-aerner/lm-v2,open-llama,skip,Load problem
+aerner/lm-v2,open-llama,skip,Load problem: maximum recursion depth exceeded while calling a Python object
 afonsosamarques/ardt-vanilla-combo_train_hopper_v2-2508_1336-33,decision_transformer,skip,Load problem
 aihijo/gec-zh-gector-bert-large,gector,skip,Load problem
 albert-base-v2,albert
@@ -72,7 +72,7 @@ facebook/esm2_t6_8M_UR50D,esm,xfail,Tracing error: The values for attribute 'sha
 facebook/flava-full,flava,skip,Load problem
 facebook/flava-image-codebook,flava_image_codebook,skip,Load problem
 facebook/m2m100_418M,m2m_100
-facebook/mask2former-swin-base-coco-panoptic,mask2former,skip,Load problem
+facebook/mask2former-swin-base-coco-panoptic,mask2former,xfail,Accuracy validation failed
 facebook/maskformer-swin-base-coco,maskformer
 facebook/mms-tts-eng,vits,skip,Load problem
 facebook/musicgen-small,musicgen,skip,Load problem
@@ -92,7 +92,7 @@ fusing/latent-diffusion-text2im-large,ldmbert,skip,Load problem
 Geor111y/flair-ner-addresses-extractor,flair,skip,Load problem
 gia-project/gia,gia,skip,Load problem
 gokuls/bert_12_layer_model_v1,hybridbert,skip,Load problem
-google/bigbird-roberta-base,big_bird,skip,Load problem
+google/bigbird-roberta-base,big_bird
 google/bit-50,bit
 google/canine-s,canine,xfail,aten::slice: Parameter axis 3 out of the tensor rank range
 google/efficientnet-b2,efficientnet,xfail,Compile error: AvgPool: Kernel after dilation has size (dim: 1408) larger than the data shape after padding (dim: 9) at axis 0.
@@ -127,10 +127,10 @@ hf-internal-testing/tiny-random-DonutSwinModel,donut-swin,xfail,Unsupported op a
 hf-internal-testing/tiny-random-EfficientFormerForImageClassification,efficientformer
 hf-internal-testing/tiny-random-flaubert,flaubert
 hf-internal-testing/tiny-random-FocalNetModel,focalnet,xfail,Unsupported op aten::adaptive_avg_pool1d
-hf-internal-testing/tiny-random-GPTBigCodeForCausalLM,gpt_bigcode,skip,Load problem
+hf-internal-testing/tiny-random-GPTBigCodeForCausalLM,gpt_bigcode,xfail,Conversion is failed for: aten::mul
 hf-internal-testing/tiny-random-GPTJModel,gptj
 hf-internal-testing/tiny-random-groupvit,groupvit
-hf-internal-testing/tiny-random-IBertModel,ibert,skip,Load problem
+hf-internal-testing/tiny-random-IBertModel,ibert
 hf-internal-testing/tiny-random-ImageGPTModel,imagegpt
 hf-internal-testing/tiny-random-LevitModel,levit,skip,Load problem
 hf-internal-testing/tiny-random-LiltModel,lilt,skip,Load problem
@@ -309,7 +309,7 @@ regisss/bridgetower-newyorker-a100-8x,bridgetower
 rinna/japanese-cloob-vit-b-16,cloob,skip,Load problem
 Rocketknight1/tiny-random-falcon-7b,falcon,skip,Load problem
 RUCAIBox/mass-base-uncased,mass,skip,Load problem
-RWKV/rwkv-4-169m-pile,rwkv,skip,Load problem
+RWKV/rwkv-4-169m-pile,rwkv,xfail,Unsupported op aten::maximum
 sahasrarjn/interbert,BERT,skip,Load problem
 saibo/genkalm-medium-gpt2,genkalm,skip,Load problem
 SajjadAyoubi/clip-fa-vision,clip_vision_model

tests/model_hub_tests/torch_tests/test_timm.py

@@ -0,0 +1,67 @@
+# Copyright (C) 2018-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import timm
+import torch
+import pytest
+from models_hub_common.test_convert_model import TestConvertModel
+from models_hub_common.constants import hf_hub_cache_dir
+from models_hub_common.utils import cleanup_dir
+from openvino import convert_model
+
+
+def get_all_models() -> list:
+    m_list = timm.list_pretrained()
+    return m_list
+
+
+# To make tests reproducible we seed the random generator
+torch.manual_seed(0)
+
+
+class TestTimmConvertModel(TestConvertModel):
+    def load_model(self, model_name, model_link):
+        m = timm.create_model(model_name, pretrained=True)
+        cfg = timm.get_pretrained_cfg(model_name)
+        shape = [1] + list(cfg.input_size)
+        self.example = (torch.randn(shape),)
+        self.inputs = (torch.randn(shape),)
+        return m
+
+    def get_inputs_info(self, model_obj):
+        return None
+
+    def prepare_inputs(self, inputs_info):
+        return [i.numpy() for i in self.inputs]
+
+    def convert_model(self, model_obj):
+        ov_model = convert_model(model_obj, example_input=self.example)
+        return ov_model
+
+    def infer_fw_model(self, model_obj, inputs):
+        fw_outputs = model_obj(*[torch.from_numpy(i) for i in inputs])
+        if isinstance(fw_outputs, dict):
+            for k in fw_outputs.keys():
+                fw_outputs[k] = fw_outputs[k].numpy(force=True)
+        elif isinstance(fw_outputs, (list, tuple)):
+            fw_outputs = [o.numpy(force=True) for o in fw_outputs]
+        else:
+            fw_outputs = [fw_outputs.numpy(force=True)]
+        return fw_outputs
+
+    def teardown_method(self):
+        # remove all downloaded files from cache
+        cleanup_dir(hf_hub_cache_dir)
+        super().teardown_method()
+
+    @pytest.mark.parametrize("name", ["mobilevitv2_050.cvnets_in1k",
+                                      "poolformerv2_s12.sail_in1k",
+                                      "vit_base_patch8_224.augreg_in21k"])
+    @pytest.mark.precommit
+    def test_convert_model_precommit(self, name, ie_device):
+        self.run(name, None, ie_device)
+
+    @pytest.mark.parametrize("name", get_all_models())
+    @pytest.mark.nightly
+    def test_convert_model_all_models(self, name, ie_device):
+        self.run(name, None, ie_device)

tests/model_hub_tests/torch_tests/test_transformers.py

@@ -1,14 +1,14 @@
 # Copyright (C) 2018-2023 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-import pytest
 import os
+import pytest
 import torch
-import tempfile
 from huggingface_hub import model_info
 from models_hub_common.test_convert_model import TestConvertModel
 from openvino import convert_model
-from models_hub_common.utils import get_models_list
+from models_hub_common.utils import get_models_list, cleanup_dir
+from models_hub_common.constants import hf_hub_cache_dir
 
 
 def flattenize_tuples(list_input):
@@ -37,15 +37,10 @@ class TestTransformersModel(TestConvertModel):
     def setup_class(self):
         from PIL import Image
         import requests
-        import os
 
         url = "http://images.cocodataset.org/val2017/000000039769.jpg"
         self.image = Image.open(requests.get(url, stream=True).raw)
 
-        self.cache_dir = tempfile.TemporaryDirectory()
-        print(f"Setting temp cache dir: {str(self.cache_dir)}")
-        os.environ["HUGGINGFACE_HUB_CACHE"] = str(self.cache_dir)
-
     def load_model(self, name, type):
         mi = model_info(name)
         auto_processor = None
@@ -243,6 +238,12 @@ class TestTransformersModel(TestConvertModel):
             from transformers import AutoModel
             model = AutoModel.from_pretrained(name, torchscript=True)
         self.example = example
+        model.eval()
+        # do first inference
+        if isinstance(example, dict):
+            model(**example)
+        else:
+            model(*example)
         return model
 
     def get_inputs_info(self, model_obj):
@@ -267,9 +268,8 @@ class TestTransformersModel(TestConvertModel):
         return flattenize_outputs(fw_outputs)
 
     def teardown_method(self):
-        # cleanup tmpdir
-        print(f"Cleaning temp cache dir: {str(self.cache_dir)}")
-        self.cache_dir.cleanup()
+        # remove all downloaded files from cache
+        cleanup_dir(hf_hub_cache_dir)
         super().teardown_method()
 
     @pytest.mark.parametrize("name,type", [("bert-base-uncased", "bert"),