IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[POT] Tests & References update (#9694)

Browse Source 
* Enable disabled tests

* Update references

* Update references

* Update passes.py for FP16 graph

* Update precision for nightly mobilenet

* Update references

* Update refs and disable model

* Update nightly refs

* Update nightly

* Revert mbv2 sanity gold
This commit is contained in:
Nikita Malinin
2022-01-20 16:23:00 +03:00
committed by GitHub
parent 8f3d030d79
commit ed634bd18e
28 changed files with 84 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
tools/pot/openvino/tools/pot/graph/passes.py
View File

@@ -708,7 +708,8 @@ def create_bias_node(graph: Graph, src_node):
for destination_port in destination_ports:
add_op.out_port(0).connect(destination_port)
add_bias.out_node(0)['Insert_Convert_operation_after'] = True
if bias_dtype != np.float32:
add_bias.out_node(0)['Insert_Convert_operation_after'] = True
def create_fake_quantize_node(graph: Graph, name, data_type=np.float32):

4
tools/pot/tests/data/reference_models/densenet121_example_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:911e5dc627106a842f78fcb14651de8fb9244f1adb5b75951cf923fc488692e4
size 1085988
oid sha256:cb5c76c82caec54e07eeac7da41f017707ede198f42ebed7d654021238a5646c
size 1141705

4
tools/pot/tests/data/reference_models/efficientnet_b0_example_ig_pt_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:181402cfe46282cf96d82f1e0b68f2ba5ccbcaccbb1ce3e712fd1b4cb5883917
size 463708
oid sha256:9ae7b4c9c297bb92b2793b0dd9b7b41ce636524012aab8695fa5a398648a6da3
size 494690

4
tools/pot/tests/data/reference_models/googlenet_example_ig_params_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c21805660b9cda4bf2335ee57a4a503f56e661ff4495e0a50e13e26489bcf508
size 265068
oid sha256:4b020df81f26b23e38b72000e3bc67872e8a6dbc30ef2c881131d5b16273db49
size 281898

4
tools/pot/tests/data/reference_models/googlenet_example_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:0fdfc311244ab6cb8d9f1791f630f55532269f3eab2871c5b49f52d498a67dc8
size 266701
oid sha256:6f9fbf518218c6019b4b890b6d17a96b45f57a85dfd1fa0579fc88ef933036d6
size 283531

4
tools/pot/tests/data/reference_models/mobilenet_v3_small_example_ig_pt_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f4091df7f399097a4ecb046014df916b26c79bdef30ff564d349f314a61de217
size 365966
oid sha256:6ff77d9a841a8d5f85d8cf9d9aba7ceb4110a5d71e370570511bb7c69897c3d0
size 388486

4
tools/pot/tests/data/reference_models/mobilenet_v3_small_example_stat_mean_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:6f2cea1cb0c85f3b3b5d360ffa383f91cebadb2cd6696a524bbec16e1e780fe4
size 200424
oid sha256:2b430c98eb022f454c6a244997d12f2bc184183f27f3a99bfeafb2217f23dc5e
size 224454

4
tools/pot/tests/data/reference_models/mobilenetv2_example_ig_params_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:6daf70c6345374e8b9c7909db2ffa4975143f2335b1d0813566ebe11ee6017e5
size 249292
oid sha256:ec69d25d8444a6a6d3279a49a44329f232fb38bdb8070d9390f30d7baabcbbe4
size 260800

4
tools/pot/tests/data/reference_models/mobilenetv2_example_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:420901896a67c463a3e1b872d832f0663f3dcc68809e70ed1df4404969438f63
size 262249
oid sha256:550e36e68683089a8399b87a0f39a3051e64853a9a0e453abf7be2fb02603080
size 273729

4
tools/pot/tests/data/reference_models/mobilenetv2_example_stat_mean_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:eaddc014b3fd47d6e46a50c7f79365d6c2210a921418650a72df3bb645619043
size 102427
oid sha256:f9a59f3c9a5b9a0dc6603bc864c9f4f0458c7bd7f5ab971862a4fd5dea859959
size 113934

4
tools/pot/tests/data/reference_models/mobilenetv2_ssd_example_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d9dc15333a2c46d9c3734a6408156e3b0edac09c2f8c0ef761ae797ef1783000
size 427809
oid sha256:edf93c51608af383936da2074835536992cdd42b9c451d68290a808f47e14b2f
size 456015

4
tools/pot/tests/data/reference_models/mobilenetv2_ssd_example_stat_mean_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c8536f0913590f6db45864b610f4c3ede34a0370baccfd8c370a2c4e6fec5405
size 315401
oid sha256:7c74bddf66718cf963c507a3aa0c8f1efe0b5ed08414ea6f5a01efb8dbd106c4
size 343757

3
tools/pot/tests/data/reference_models/mtcnn_ig_params_onet_caffe.xml Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a756a350fdceae199a797bcf254f8c7b6ec73976367f4654635fe42a26ef5039
size 48284

3
tools/pot/tests/data/reference_models/mtcnn_ig_params_pnet_caffe.xml Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:5d6d0149bb449eaac1c4ffb01394d497464f411277366d708199a963ab91dc99
size 30173

3
tools/pot/tests/data/reference_models/mtcnn_ig_params_rnet_caffe.xml Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:eaf9864e73d4dba5b98632a00f338a853158ee207a9eb0ab9fe39308c03cd189
size 27879

4
tools/pot/tests/data/reference_models/resnet_example_ig_params_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:2c0be4c6d55c04b7bd84aa49cba99926f30cff00941cafdf25d795ed77a869f0
size 97284
oid sha256:51f06b88c870bd66f700ecdbe994abafd5f1f7a3e9b28cad17332c2223086091
size 103689

4
tools/pot/tests/data/reference_models/resnet_example_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d8fe024f86ca61a1665988516117766a7e373b27dcaa0ebf6187592203b0bf9c
size 115224
oid sha256:d031424b6c9d8cd432712f1afeccdd591262b2f90f04f6c38de1f25780123df1
size 121601

4
tools/pot/tests/data/reference_models/resnet_example_ranger_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:1965fbfb9784b6047f329fd80a50230e3fd937582f17ce0118a6800717b4a082
size 54282
oid sha256:94e83c262b3fb0b6bc0fc6279e1f19fb59c853f261d1b1bbb9334477c837351a
size 60629

4
tools/pot/tests/data/reference_models/resnet_example_stat_abs_max_max_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:5f988ff4b8d6152ec91b6f9f3338ee5232cca2025f3ef488529e3ab19b8c5749
size 100923
oid sha256:125a6186628f8fc57de8677667baa554856d883ebd72b367ae6cb2b82b69e425
size 108569

4
tools/pot/tests/data/reference_models/resnet_example_stat_abs_max_min_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4022b4e2715b1e16ec1d5baad6a3b6f7a81a9605704ae3ab07a83e3ea3acfcf8
size 103270
oid sha256:d66145d226fcb4244987b9c2a2296eab5be9c5f7fc936dccfe6c255d6254ae96
size 110824

4
tools/pot/tests/data/reference_models/resnet_example_stat_max_min_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d314963df178e97a6abc885cbfa79f2a1f5e0ff38ca295735e3931bab458732f
size 93279
oid sha256:c78e42577f381d5cc2d860d8e5ce67d650fc3f3ee9fe2d20cafa1fe4993ad21c
size 100679

4
tools/pot/tests/data/reference_models/resnet_example_stat_min_quantile_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c1bc62470a97ada5b5614d9fad57b65db71c83d1c5608c3a8163763322c48136
size 69528
oid sha256:3aa2b0d0dec8041537c874d2b4b0a014654f786c9e8b949ffd71f653a5bd1cf2
size 76474

4
tools/pot/tests/data/reference_models/resnet_example_stat_quantile_abs_quantile_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:24cf8d0c7b73a78d2232a3553e7acf266fe16eb954ff7421007360bd97069c21
size 48137
oid sha256:ef30e3f18abcfdd18dd68d02d75bedf9477b10fd2176e1adbd7f4fb4fbb52a43
size 54512

4
tools/pot/tests/data/reference_models/squeezenet1_1_example_stat_mean_pytorch.xml
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c558770b2a07b687574c74ca644f16d6d0d126476f6a2a1678a304376a1e602c
size 113250
oid sha256:e4cbf76076d3f0d315688321182af64c0f5746e6c0d4f17065a3f7c3cda88423
size 124297

17
tools/pot/tests/test_graph.py
View File

@@ -23,8 +23,7 @@ TEST_MODELS = [
('googlenet_example', 'pytorch', 'ANY'),
('mobilenetv2_ssd_example', 'pytorch', 'ANY'),
('densenet121_example', 'pytorch', 'ANY'),
# TODO: Enable these tests after solving IRReader problem
# ('multiple_out_ports_net', 'tf', 'ANY'),
('multiple_out_ports_net', 'tf', 'ANY'),
('lstm_example', 'pytorch', 'GNA'),
('multiple_outputs_net_example', 'dldt', 'GNA'),
# ('tensor_iterator_example', 'tf', 'ANY'),
@@ -60,7 +59,7 @@ MODELS_FOR_TESTING_IGNORED_PARAMS = [
('mobilenetv2_example', 'pytorch'),
('resnet_example', 'pytorch'),
('googlenet_example', 'pytorch'),
# ('mtcnn', 'caffe')
('mtcnn', 'caffe')
]
@@ -229,6 +228,8 @@ def test_build_quantization_graph_with_ignored_blocks(tmp_path, models, model_na
def test_multibranch_propagation_without_fq_moving():
# TODO: Enable this test after IRReader solve the problem with MaxPool #9613
pytest.skip()
TEST_CASES_PATH = TEST_ROOT / 'data' / 'test_cases_refs'
model_path = (TEST_CASES_PATH / 'test_ig_border_case_without_fq_moving.xml').as_posix()
weights_path = (TEST_CASES_PATH / 'test_ig_border_case_without_fq_moving.bin').as_posix()
@@ -252,13 +253,13 @@ def test_multibranch_propagation_without_fq_moving():
MODELS_WITH_LSTM = [
('lstm_example', 'pytorch', {
'LSTM_15/TensorIterator/22/variable_1':
['Assign_298'],
['Assign_304'],
'LSTM_15/TensorIterator/24/variable_2':
['Assign_305'],
['Assign_311'],
'LSTM_19/TensorIterator/22/variable_1':
['Assign_327'],
['Assign_333'],
'LSTM_19/TensorIterator/24/variable_2':
['Assign_334']
['Assign_340']
})
]
@@ -277,6 +278,8 @@ def test_lstm_ends(tmp_path, models):
def test_multibranch_propagation_with_fq_moving():
# TODO: Enable this test after IRReader solve the problem with MaxPool #9613
pytest.skip()
TEST_CASES_PATH = TEST_ROOT / 'data' / 'test_cases_refs'
model_path = (TEST_CASES_PATH / 'test_ig_border_case_with_fq_moving.xml').as_posix()
weights_path = (TEST_CASES_PATH / 'test_ig_border_case_with_fq_moving.bin').as_posix()

31
tools/pot/tests/test_nightly.py
View File

@@ -14,10 +14,9 @@ from .utils.config import get_engine_config, merge_configs
init_logger(level='INFO')
TEST_MODELS_DEFAULT = [
# TODO: Enable this test after solving runtime (segfault) bug
# ('ssd512', 'caffe', 'FP16', {
# 'performance': {'map': 0.9088},
# 'mixed': {'map': 0.904}}),
('ssd512', 'caffe', 'FP32', {
'performance': {'map': 0.9088},
'mixed': {'map': 0.904}}),
('googlenet-v3', 'tf', 'FP32', {
'performance': {'accuracy@top1': 0.7793, 'accuracy@top5': 0.9378},
'mixed': {'accuracy@top1': 0.7793, 'accuracy@top5': 0.9378}}),
@@ -27,7 +26,7 @@ TEST_MODELS_DEFAULT = [
('mobilenet-v1-1.0-224-tf', 'tf', 'FP32', {
'performance': {'accuracy@top1': 0.70896, 'accuracy@top5': 0.89792},
'mixed': {'accuracy@top1': 0.70922, 'accuracy@top5': 0.89806}}),
('mobilenet-v2-pytorch', 'pytorch', 'FP32', {
('mobilenet-v2-pytorch', 'pytorch', 'FP16', {
'performance': {'accuracy@top1': 0.71552, 'accuracy@top5': 0.90222},
'mixed': {'accuracy@top1': 0.71512, 'accuracy@top5': 0.90172}}),
('resnet-50-pytorch', 'pytorch', 'FP32', {
@@ -36,26 +35,24 @@ TEST_MODELS_DEFAULT = [
('googlenet-v3-pytorch', 'pytorch', 'FP32', {
'performance': {'accuracy@top1': 0.77562, 'accuracy@top5': 0.9363},
'mixed': {'accuracy@top1': 0.77562, 'accuracy@top5': 0.9363}}),
('densenet-121', 'caffe', 'FP32', {
'performance': {'accuracy@top1': 0.73908, 'accuracy@top5': 0.91728},
'mixed': {'accuracy@top1': 0.7389, 'accuracy@top5': 0.91714}}),
# TODO: Enable these tests after OMZ update
# ('mobilenet-ssd', 'caffe', 'FP32', {
# 'performance': {'map': 0.71978},
# 'mixed': {'map': 0.71931}}),
# ('densenet-121', 'caffe', 'FP32', {
# 'performance': {'accuracy@top1': 0.73908, 'accuracy@top5': 0.91728},
# 'mixed': {'accuracy@top1': 0.7389, 'accuracy@top5': 0.91714}}),
('mobilenet-ssd', 'caffe', 'FP32', {
'performance': {'map': 0.666},
'mixed': {'map': 0.664}}),
('octave-resnet-26-0.25', 'mxnet', 'FP32', {
'performance': {'accuracy@top1': 0.7581, 'accuracy@top5': 0.9256},
'mixed': {'accuracy@top1': 0.759, 'accuracy@top5': 0.92466}}),
# TODO: Enable these tests after solving FP16 bug
# ('ssd_mobilenet_v1_coco', 'tf', 'FP16', {
# 'performance': {'coco_precision': 0.2312},
# 'mixed': {'coco_precision': 0.2314}})
('ssd_mobilenet_v1_coco', 'tf', 'FP16', {
'performance': {'coco_precision': 0.2312},
'mixed': {'coco_precision': 0.2314}})
]
TEST_MODELS_ACC_AWARE = [
('efficientnet-b0-pytorch', 'pytorch', 'CPU', {'performance': {'accuracy@top1': 0.7663,
'accuracy@top5': 0.9294}}),
# ('mobilenet-ssd', 'caffe', 'CPU', {'performance': {'map': 0.7222}}),
('mobilenet-ssd', 'caffe', 'CPU', {'performance': {'map': 0.7222}}),
('ssd512', 'caffe', 'CPU', {'performance': {'map': 0.7917}}),
('mobilenet-v1-0.25-128', 'tf', 'GNA', {'performance': {'accuracy@top1': 0.4133, 'accuracy@top5': 0.6626}})
]

19
tools/pot/tests/test_sanity.py
View File

@@ -39,23 +39,22 @@ TEST_MODELS = [
'accuracy@top5': 0.911},
{}, 'CPU'),
# ('mobilenet-ssd', 'caffe', 'AccuracyAwareQuantization', 'performance', 300, {'map': 0.7215},
# {'metric_subset_ratio': 1.0, 'max_iter_num': 1, 'metrics': [{'name': 'map', 'baseline_value': 0.7311}]}, 'CPU'),
('mobilenet-ssd', 'caffe', 'AccuracyAwareQuantization', 'performance', 300, {'map': 0.674},
{'metric_subset_ratio': 1.0, 'max_iter_num': 1, 'metrics': [{'name': 'map', 'baseline_value': 0.669}]}, 'CPU'),
# ('mobilenet-ssd', 'caffe', 'AccuracyAwareQuantization', 'performance', 300, {'map': 0.7215},
# {'metric_subset_ratio': 1.0, 'max_iter_num': 1, 'tune_hyperparams': True,
# 'metrics': [{'name': 'map', 'baseline_value': 0.7311}]}, 'CPU'),
('mobilenet-ssd', 'caffe', 'AccuracyAwareQuantization', 'performance', 300, {'map': 0.674},
{'metric_subset_ratio': 1.0, 'max_iter_num': 1, 'tune_hyperparams': True,
'metrics': [{'name': 'map', 'baseline_value': 0.669}]}, 'CPU'),
# ('mobilenet-v1-0.25-128', 'tf', 'AccuracyAwareQuantization', 'performance', 100,
# {'accuracy@top1': 0.424, 'accuracy@top5': 0.65},
# {'drop_type': 'relative', 'max_iter_num': 1, 'accuracy_drop': 0.005, 'metrics': [
# {'name': 'accuracy@top1', 'baseline_value': 0.431}]}, 'GNA'),
# This test is not able to run due to OV API problem
# ('mtcnn', 'caffe', 'DefaultQuantization', 'performance', 1, {'recall': 0.76, 'map': 0.6844}, {}, 'CPU'),
('mtcnn', 'caffe', 'DefaultQuantization', 'performance', 1, {'recall': 0.76, 'map': 0.6844}, {}, 'CPU'),
# ('mtcnn', 'caffe', 'DefaultQuantization', 'performance', 2, {'recall': 0.8, 'map': 0.7445},
# {'use_fast_bias': False}, 'CPU')
('mtcnn', 'caffe', 'DefaultQuantization', 'performance', 2, {'recall': 0.8, 'map': 0.7445},
{'use_fast_bias': False}, 'CPU')
]
CASCADE_MAP = Dict({
'mtcnn': {
@@ -254,7 +253,7 @@ TEST_MULTIPLE_OUT_PORTS = [('multiple_out_ports_net', 'tf')]
'model_name, model_framework', TEST_MULTIPLE_OUT_PORTS,
ids=['{}_{}'.format(m[0], m[1]) for m in TEST_MULTIPLE_OUT_PORTS])
def test_multiport_outputs_model(tmp_path, models, model_name, model_framework):
# TODO: Enable these tests after solving IRReader problem
# This test is not able to run due to NHWC shape that is not supported
pytest.skip()
test_dir = Path(__file__).parent
# one image as dataset

2
tools/pot/tests/test_sparsity.py
View File

@@ -39,8 +39,6 @@ TEST_SPARSITY_ALGO = [
@pytest.mark.parametrize('test_models', TEST_SPARSITY_ALGO,
ids=['{}_{}_{}_{}'.format(*m) for m in TEST_SPARSITY_ALGO])
def test_sparsity_algo(test_models, tmp_path, models):
# TODO: Enable these tests after solving IRReader problem
pytest.skip()
model_name, model_framework, algorithm, sparsity_level, normed_threshold, ref_name = test_models
algorithm_config = Dict({
'algorithms': [{