[PT FE] Add tests for detectron2 models (#19888)

* [PT FE] Add tests for detectron2 models

* Fix names of tests

* Apply suggestions from code review

Co-authored-by: Roman Kazantsev <roman.kazantsev@intel.com>

* Create secondary requirements file

---------

Co-authored-by: Roman Kazantsev <roman.kazantsev@intel.com>

.github/workflows/linux.yml

@@ -912,6 +912,7 @@ jobs:
       - name: PyTorch Models Tests
         run: |
           python3 -m pip install -r ${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}/torch_tests/requirements.txt
+          python3 -m pip install -r ${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}/torch_tests/requirements_secondary.txt
           export PYTHONPATH=${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}:$PYTHONPATH
           python3 -m pytest ${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}/torch_tests/ -m ${{ env.TYPE }} --html=${{ env.INSTALL_TEST_DIR }}/TEST-torch_model_tests.html --self-contained-html
         env:

tests/model_hub_tests/models_hub_common/utils.py

@@ -32,7 +32,8 @@ def compare_two_tensors(ov_res, fw_res, eps):
     is_ok = True
     if not np.allclose(ov_res, fw_res, atol=eps, rtol=eps, equal_nan=True):
         is_ok = False
-        print("Max diff is {}".format(np.array(abs(ov_res - fw_res)).max()))
+        max_diff = np.abs(ov_res.astype(np.float32) - fw_res.astype(np.float32)).max()
+        print("Max diff is {}".format(max_diff))
     else:
         print("Accuracy validation successful!\n")
         print("absolute eps: {}, relative eps: {}".format(eps, eps))

tests/model_hub_tests/torch_tests/detectron2_models

@@ -0,0 +1,65 @@
+COCO-Detection/fast_rcnn_R_50_FPN_1x,none
+COCO-Detection/faster_rcnn_R_101_C4_3x,none
+COCO-Detection/faster_rcnn_R_101_DC5_3x,none
+COCO-Detection/faster_rcnn_R_101_FPN_3x,none
+COCO-Detection/faster_rcnn_R_50_C4_1x,none
+COCO-Detection/faster_rcnn_R_50_C4_3x,none
+COCO-Detection/faster_rcnn_R_50_DC5_1x,none
+COCO-Detection/faster_rcnn_R_50_DC5_3x,none
+COCO-Detection/faster_rcnn_R_50_FPN_1x,none
+COCO-Detection/faster_rcnn_R_50_FPN_3x,none
+COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x,none
+COCO-Detection/retinanet_R_101_FPN_3x,none
+COCO-Detection/retinanet_R_50_FPN_1x,none
+COCO-Detection/retinanet_R_50_FPN_3x,none
+COCO-Detection/rpn_R_50_C4_1x,none
+COCO-Detection/rpn_R_50_FPN_1x,none
+COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x,none
+COCO-InstanceSegmentation/mask_rcnn_R_101_C4_3x,none
+COCO-InstanceSegmentation/mask_rcnn_R_101_DC5_3x,none
+COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x,none
+COCO-InstanceSegmentation/mask_rcnn_R_50_C4_1x,none
+COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x,none
+COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_1x,none
+COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_3x,none
+#COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x_giou,none - Pretrained model is not available!
+COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x,none
+COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x,none
+#COCO-Keypoints/Base-Keypoint-RCNN-FPN,none - Pretrained model is not available!
+COCO-Keypoints/keypoint_rcnn_R_101_FPN_3x,none
+COCO-Keypoints/keypoint_rcnn_R_50_FPN_1x,none
+COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x,none
+COCO-Keypoints/keypoint_rcnn_X_101_32x8d_FPN_3x,none
+#COCO-PanopticSegmentation/Base-Panoptic-FPN,none - Pretrained model is not available!
+COCO-PanopticSegmentation/panoptic_fpn_R_101_3x,none
+COCO-PanopticSegmentation/panoptic_fpn_R_50_1x,none
+COCO-PanopticSegmentation/panoptic_fpn_R_50_3x,none
+Cityscapes/mask_rcnn_R_50_FPN,none
+Detectron1-Comparisons/faster_rcnn_R_50_FPN_noaug_1x,none
+Detectron1-Comparisons/keypoint_rcnn_R_50_FPN_1x,none
+Detectron1-Comparisons/mask_rcnn_R_50_FPN_noaug_1x,none
+LVISv0.5-InstanceSegmentation/mask_rcnn_R_101_FPN_1x,none
+LVISv0.5-InstanceSegmentation/mask_rcnn_R_50_FPN_1x,none
+LVISv0.5-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_1x,none
+#LVISv1-InstanceSegmentation/mask_rcnn_R_101_FPN_1x,none - Pretrained model is not available!
+#LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x,none - Pretrained model is not available!
+#LVISv1-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_1x,none - Pretrained model is not available!
+#Misc/mask_rcnn_R_50_FPN_1x_cls_agnostic,none - Pretrained model is not available!
+Misc/cascade_mask_rcnn_R_50_FPN_1x,none
+Misc/cascade_mask_rcnn_R_50_FPN_3x,none
+Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv,none
+Misc/mask_rcnn_R_50_FPN_1x_dconv_c3-c5,none
+Misc/mask_rcnn_R_50_FPN_3x_dconv_c3-c5,none
+Misc/mask_rcnn_R_50_FPN_3x_gn,none
+Misc/mask_rcnn_R_50_FPN_3x_syncbn,none
+Misc/panoptic_fpn_R_101_dconv_cascade_gn_3x,none
+Misc/scratch_mask_rcnn_R_50_FPN_3x_gn,none
+Misc/scratch_mask_rcnn_R_50_FPN_9x_gn,none
+Misc/scratch_mask_rcnn_R_50_FPN_9x_syncbn,none
+#Misc/semantic_R_50_FPN_1x,none - Pretrained model is not available!
+PascalVOC-Detection/faster_rcnn_R_50_C4,none
+#PascalVOC-Detection/faster_rcnn_R_50_FPN,none - Pretrained model is not available!
+#Base-RCNN-C4,none - Pretrained model is not available!
+#Base-RCNN-DilatedC5,none - Pretrained model is not available!
+#Base-RCNN-FPN,none - Pretrained model is not available!
+#Base-RetinaNet,none - Pretrained model is not available!

tests/model_hub_tests/torch_tests/detectron2_precommit

@@ -0,0 +1,26 @@
+COCO-Detection/faster_rcnn_R_50_C4_1x,none
+COCO-Detection/faster_rcnn_R_50_DC5_3x,none
+COCO-Detection/faster_rcnn_R_50_FPN_1x,none
+COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x,none
+COCO-Detection/retinanet_R_50_FPN_1x,none
+COCO-Detection/rpn_R_50_C4_1x,none
+COCO-Detection/rpn_R_50_FPN_1x,none
+COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x,none
+COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x,none
+COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_3x,none
+COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x,none
+COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x,none
+COCO-Keypoints/keypoint_rcnn_X_101_32x8d_FPN_3x,none
+Cityscapes/mask_rcnn_R_50_FPN,none
+Detectron1-Comparisons/faster_rcnn_R_50_FPN_noaug_1x,none
+Detectron1-Comparisons/keypoint_rcnn_R_50_FPN_1x,none
+Detectron1-Comparisons/mask_rcnn_R_50_FPN_noaug_1x,none
+LVISv0.5-InstanceSegmentation/mask_rcnn_R_50_FPN_1x,none
+LVISv0.5-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_1x,none
+Misc/cascade_mask_rcnn_R_50_FPN_3x,none
+Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv,none
+Misc/mask_rcnn_R_50_FPN_3x_dconv_c3-c5,none
+Misc/mask_rcnn_R_50_FPN_3x_gn,none
+Misc/mask_rcnn_R_50_FPN_3x_syncbn,none
+Misc/scratch_mask_rcnn_R_50_FPN_9x_syncbn,none
+PascalVOC-Detection/faster_rcnn_R_50_C4,none

tests/model_hub_tests/torch_tests/requirements.txt

@@ -4,4 +4,4 @@ pytest
 pytest-html
 torch
 torchvision
-av
+av

tests/model_hub_tests/torch_tests/requirements_secondary.txt

@@ -0,0 +1,3 @@
+-c ../../constraints.txt
+# This file contains requirements dependednt from modules in requirements.txt
+git+https://github.com/facebookresearch/detectron2.git 

tests/model_hub_tests/torch_tests/test_detectron2.py

@@ -0,0 +1,93 @@
+# Copyright (C) 2018-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import pytest
+import torch
+from models_hub_common.test_convert_model import TestConvertModel
+from openvino import convert_model
+from models_hub_common.utils import get_models_list, compare_two_tensors
+
+
+class TestDetectron2ConvertModel(TestConvertModel):
+    def setup_class(self):
+        from PIL import Image
+        import requests
+
+        url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+        self.image = Image.open(requests.get(url, stream=True).raw)
+        self.image = self.image.resize([640, 480])
+
+    def load_model(self, model_name, model_link):
+        from detectron2 import model_zoo, export
+        from detectron2.modeling import build_model
+        from detectron2.checkpoint import DetectionCheckpointer
+        from detectron2.config import CfgNode
+        import torchvision.transforms as transforms
+
+        transform = transforms.Compose([transforms.PILToTensor()])
+        image = transform(self.image)
+
+        cfg = model_zoo.get_config(model_name + ".yaml", trained=True)
+        assert isinstance(cfg, CfgNode), "Unexpected config"
+        cfg.MODEL.DEVICE = "cpu"
+        model = build_model(cfg)
+        DetectionCheckpointer(model, save_to_disk=False).load(cfg.MODEL.WEIGHTS)
+        
+        model.eval()
+        inputs = [{"image": image,
+                   "height": torch.tensor(image.shape[1]),
+                   "width": torch.tensor(image.shape[2])}]
+        adapter = export.TracingAdapter(model, inputs)
+
+        self.example = adapter.flattened_inputs
+        return adapter
+
+    def get_inputs_info(self, model_obj):
+        return None
+
+    def prepare_inputs(self, inputs_info):
+        return [i.numpy() for i in self.example]
+
+    def convert_model(self, model_obj):
+        ov_model = convert_model(model_obj, example_input=self.example)
+        return ov_model
+
+    def infer_fw_model(self, model_obj, inputs):
+        fw_outputs = model_obj(*[torch.from_numpy(i) for i in inputs])
+        if isinstance(fw_outputs, dict):
+            for k in fw_outputs.keys():
+                fw_outputs[k] = fw_outputs[k].numpy(force=True)
+        elif isinstance(fw_outputs, (list, tuple)):
+            fw_outputs = [o.numpy(force=True) for o in fw_outputs]
+        else:
+            fw_outputs = [fw_outputs.numpy(force=True)]
+        return fw_outputs
+
+    def compare_results(self, fw_outputs, ov_outputs):
+        assert len(fw_outputs) == len(ov_outputs), \
+            "Different number of outputs between TensorFlow and OpenVINO:" \
+            " {} vs. {}".format(len(fw_outputs), len(ov_outputs))
+
+        fw_eps = 5e-2
+        is_ok = True
+        for i in range(len(ov_outputs)):
+            cur_fw_res = fw_outputs[i]
+            cur_ov_res = ov_outputs[i]
+            l = min(len(cur_fw_res), len(cur_ov_res))
+            assert l > 0 or len(cur_fw_res) == len(cur_ov_res), "No boxes were selected."
+            print(f"fw_re: {cur_fw_res};\n ov_res: {cur_ov_res}")
+            is_ok = compare_two_tensors(cur_ov_res[:l], cur_fw_res[:l], fw_eps)
+        assert is_ok, "Accuracy validation failed"
+
+    @pytest.mark.parametrize("name,type,mark,reason",
+                             get_models_list(os.path.join(os.path.dirname(__file__), "detectron2_precommit")))
+    @pytest.mark.precommit
+    def test_detectron2_precommit(self, name, type, mark, reason, ie_device):
+        self.run(name, None, ie_device)
+
+    @pytest.mark.parametrize("name,type,mark,reason",
+                             get_models_list(os.path.join(os.path.dirname(__file__), "detectron2_models")))
+    @pytest.mark.nightly
+    def test_detectron2_all_models(self, name, type, mark, reason, ie_device):
+        self.run(name, None, ie_device)