IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[PT FE] Add tests for detectron2 models (#19888)

Browse Source 
* [PT FE] Add tests for detectron2 models

* Fix names of tests

* Apply suggestions from code review

Co-authored-by: Roman Kazantsev <roman.kazantsev@intel.com>

* Create secondary requirements file

---------

Co-authored-by: Roman Kazantsev <roman.kazantsev@intel.com>
This commit is contained in:
Maxim Vafin 2023-09-18 13:40:40 +02:00 committed by GitHub
parent 49df8fa45e
commit a4cbac3dee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 191 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/linux.yml vendored
View File

@ -912,6 +912,7 @@ jobs:
- name: PyTorch Models Tests
run: |
python3 -m pip install -r ${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}/torch_tests/requirements.txt
python3 -m pip install -r ${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}/torch_tests/requirements_secondary.txt
export PYTHONPATH=${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}:$PYTHONPATH
python3 -m pytest ${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}/torch_tests/ -m ${{ env.TYPE }} --html=${{ env.INSTALL_TEST_DIR }}/TEST-torch_model_tests.html --self-contained-html
env:

3
tests/model_hub_tests/models_hub_common/utils.py
View File

@ -32,7 +32,8 @@ def compare_two_tensors(ov_res, fw_res, eps):
is_ok = True
if not np.allclose(ov_res, fw_res, atol=eps, rtol=eps, equal_nan=True):
is_ok = False
print("Max diff is {}".format(np.array(abs(ov_res - fw_res)).max()))
max_diff = np.abs(ov_res.astype(np.float32) - fw_res.astype(np.float32)).max()
print("Max diff is {}".format(max_diff))
else:
print("Accuracy validation successful!\n")
print("absolute eps: {}, relative eps: {}".format(eps, eps))

65
tests/model_hub_tests/torch_tests/detectron2_models Normal file
View File

@ -0,0 +1,65 @@
COCO-Detection/fast_rcnn_R_50_FPN_1x,none
COCO-Detection/faster_rcnn_R_101_C4_3x,none
COCO-Detection/faster_rcnn_R_101_DC5_3x,none
COCO-Detection/faster_rcnn_R_101_FPN_3x,none
COCO-Detection/faster_rcnn_R_50_C4_1x,none
COCO-Detection/faster_rcnn_R_50_C4_3x,none
COCO-Detection/faster_rcnn_R_50_DC5_1x,none
COCO-Detection/faster_rcnn_R_50_DC5_3x,none
COCO-Detection/faster_rcnn_R_50_FPN_1x,none
COCO-Detection/faster_rcnn_R_50_FPN_3x,none
COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x,none
COCO-Detection/retinanet_R_101_FPN_3x,none
COCO-Detection/retinanet_R_50_FPN_1x,none
COCO-Detection/retinanet_R_50_FPN_3x,none
COCO-Detection/rpn_R_50_C4_1x,none
COCO-Detection/rpn_R_50_FPN_1x,none
COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x,none
COCO-InstanceSegmentation/mask_rcnn_R_101_C4_3x,none
COCO-InstanceSegmentation/mask_rcnn_R_101_DC5_3x,none
COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x,none
COCO-InstanceSegmentation/mask_rcnn_R_50_C4_1x,none
COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x,none
COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_1x,none
COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_3x,none
#COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x_giou,none - Pretrained model is not available!
COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x,none
COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x,none
#COCO-Keypoints/Base-Keypoint-RCNN-FPN,none - Pretrained model is not available!
COCO-Keypoints/keypoint_rcnn_R_101_FPN_3x,none
COCO-Keypoints/keypoint_rcnn_R_50_FPN_1x,none
COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x,none
COCO-Keypoints/keypoint_rcnn_X_101_32x8d_FPN_3x,none
#COCO-PanopticSegmentation/Base-Panoptic-FPN,none - Pretrained model is not available!
COCO-PanopticSegmentation/panoptic_fpn_R_101_3x,none
COCO-PanopticSegmentation/panoptic_fpn_R_50_1x,none
COCO-PanopticSegmentation/panoptic_fpn_R_50_3x,none
Cityscapes/mask_rcnn_R_50_FPN,none
Detectron1-Comparisons/faster_rcnn_R_50_FPN_noaug_1x,none
Detectron1-Comparisons/keypoint_rcnn_R_50_FPN_1x,none
Detectron1-Comparisons/mask_rcnn_R_50_FPN_noaug_1x,none
LVISv0.5-InstanceSegmentation/mask_rcnn_R_101_FPN_1x,none
LVISv0.5-InstanceSegmentation/mask_rcnn_R_50_FPN_1x,none
LVISv0.5-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_1x,none
#LVISv1-InstanceSegmentation/mask_rcnn_R_101_FPN_1x,none - Pretrained model is not available!
#LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x,none - Pretrained model is not available!
#LVISv1-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_1x,none - Pretrained model is not available!
#Misc/mask_rcnn_R_50_FPN_1x_cls_agnostic,none - Pretrained model is not available!
Misc/cascade_mask_rcnn_R_50_FPN_1x,none
Misc/cascade_mask_rcnn_R_50_FPN_3x,none
Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv,none
Misc/mask_rcnn_R_50_FPN_1x_dconv_c3-c5,none
Misc/mask_rcnn_R_50_FPN_3x_dconv_c3-c5,none
Misc/mask_rcnn_R_50_FPN_3x_gn,none
Misc/mask_rcnn_R_50_FPN_3x_syncbn,none
Misc/panoptic_fpn_R_101_dconv_cascade_gn_3x,none
Misc/scratch_mask_rcnn_R_50_FPN_3x_gn,none
Misc/scratch_mask_rcnn_R_50_FPN_9x_gn,none
Misc/scratch_mask_rcnn_R_50_FPN_9x_syncbn,none
#Misc/semantic_R_50_FPN_1x,none - Pretrained model is not available!
PascalVOC-Detection/faster_rcnn_R_50_C4,none
#PascalVOC-Detection/faster_rcnn_R_50_FPN,none - Pretrained model is not available!
#Base-RCNN-C4,none - Pretrained model is not available!
#Base-RCNN-DilatedC5,none - Pretrained model is not available!
#Base-RCNN-FPN,none - Pretrained model is not available!
#Base-RetinaNet,none - Pretrained model is not available!

26
tests/model_hub_tests/torch_tests/detectron2_precommit Normal file
View File

@ -0,0 +1,26 @@
COCO-Detection/faster_rcnn_R_50_C4_1x,none
COCO-Detection/faster_rcnn_R_50_DC5_3x,none
COCO-Detection/faster_rcnn_R_50_FPN_1x,none
COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x,none
COCO-Detection/retinanet_R_50_FPN_1x,none
COCO-Detection/rpn_R_50_C4_1x,none
COCO-Detection/rpn_R_50_FPN_1x,none
COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x,none
COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x,none
COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_3x,none
COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x,none
COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x,none
COCO-Keypoints/keypoint_rcnn_X_101_32x8d_FPN_3x,none
Cityscapes/mask_rcnn_R_50_FPN,none
Detectron1-Comparisons/faster_rcnn_R_50_FPN_noaug_1x,none
Detectron1-Comparisons/keypoint_rcnn_R_50_FPN_1x,none
Detectron1-Comparisons/mask_rcnn_R_50_FPN_noaug_1x,none
LVISv0.5-InstanceSegmentation/mask_rcnn_R_50_FPN_1x,none
LVISv0.5-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_1x,none
Misc/cascade_mask_rcnn_R_50_FPN_3x,none
Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv,none
Misc/mask_rcnn_R_50_FPN_3x_dconv_c3-c5,none
Misc/mask_rcnn_R_50_FPN_3x_gn,none
Misc/mask_rcnn_R_50_FPN_3x_syncbn,none
Misc/scratch_mask_rcnn_R_50_FPN_9x_syncbn,none
PascalVOC-Detection/faster_rcnn_R_50_C4,none

2
tests/model_hub_tests/torch_tests/requirements.txt
View File

@ -4,4 +4,4 @@ pytest
pytest-html
torch
torchvision
av
av

3
tests/model_hub_tests/torch_tests/requirements_secondary.txt Normal file
View File

@ -0,0 +1,3 @@
-c ../../constraints.txt
# This file contains requirements dependednt from modules in requirements.txt
git+https://github.com/facebookresearch/detectron2.git

93
tests/model_hub_tests/torch_tests/test_detectron2.py Normal file
View File

@ -0,0 +1,93 @@
# Copyright (C) 2018-2023 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import os
import pytest
import torch
from models_hub_common.test_convert_model import TestConvertModel
from openvino import convert_model
from models_hub_common.utils import get_models_list, compare_two_tensors
class TestDetectron2ConvertModel(TestConvertModel):
def setup_class(self):
from PIL import Image
import requests
url = "http://images.cocodataset.org/val2017/000000039769.jpg"
self.image = Image.open(requests.get(url, stream=True).raw)
self.image = self.image.resize([640, 480])
def load_model(self, model_name, model_link):
from detectron2 import model_zoo, export
from detectron2.modeling import build_model
from detectron2.checkpoint import DetectionCheckpointer
from detectron2.config import CfgNode
import torchvision.transforms as transforms
transform = transforms.Compose([transforms.PILToTensor()])
image = transform(self.image)
cfg = model_zoo.get_config(model_name + ".yaml", trained=True)
assert isinstance(cfg, CfgNode), "Unexpected config"
cfg.MODEL.DEVICE = "cpu"
model = build_model(cfg)
DetectionCheckpointer(model, save_to_disk=False).load(cfg.MODEL.WEIGHTS)
model.eval()
inputs = [{"image": image,
"height": torch.tensor(image.shape[1]),
"width": torch.tensor(image.shape[2])}]
adapter = export.TracingAdapter(model, inputs)
self.example = adapter.flattened_inputs
return adapter
def get_inputs_info(self, model_obj):
return None
def prepare_inputs(self, inputs_info):
return [i.numpy() for i in self.example]
def convert_model(self, model_obj):
ov_model = convert_model(model_obj, example_input=self.example)
return ov_model
def infer_fw_model(self, model_obj, inputs):
fw_outputs = model_obj(*[torch.from_numpy(i) for i in inputs])
if isinstance(fw_outputs, dict):
for k in fw_outputs.keys():
fw_outputs[k] = fw_outputs[k].numpy(force=True)
elif isinstance(fw_outputs, (list, tuple)):
fw_outputs = [o.numpy(force=True) for o in fw_outputs]
else:
fw_outputs = [fw_outputs.numpy(force=True)]
return fw_outputs
def compare_results(self, fw_outputs, ov_outputs):
assert len(fw_outputs) == len(ov_outputs), \
"Different number of outputs between TensorFlow and OpenVINO:" \
" {} vs. {}".format(len(fw_outputs), len(ov_outputs))
fw_eps = 5e-2
is_ok = True
for i in range(len(ov_outputs)):
cur_fw_res = fw_outputs[i]
cur_ov_res = ov_outputs[i]
l = min(len(cur_fw_res), len(cur_ov_res))
assert l > 0 or len(cur_fw_res) == len(cur_ov_res), "No boxes were selected."
print(f"fw_re: {cur_fw_res};\n ov_res: {cur_ov_res}")
is_ok = compare_two_tensors(cur_ov_res[:l], cur_fw_res[:l], fw_eps)
assert is_ok, "Accuracy validation failed"
@pytest.mark.parametrize("name,type,mark,reason",
get_models_list(os.path.join(os.path.dirname(__file__), "detectron2_precommit")))
@pytest.mark.precommit
def test_detectron2_precommit(self, name, type, mark, reason, ie_device):
self.run(name, None, ie_device)
@pytest.mark.parametrize("name,type,mark,reason",
get_models_list(os.path.join(os.path.dirname(__file__), "detectron2_models")))
@pytest.mark.nightly
def test_detectron2_all_models(self, name, type, mark, reason, ie_device):
self.run(name, None, ie_device)