From 77794535ab904bc6ba28a6f0a9e449869f00e4b2 Mon Sep 17 00:00:00 2001 From: Jan Iwaszkiewicz Date: Thu, 22 Oct 2020 14:06:23 +0200 Subject: [PATCH] [ONNX] WA for I64 images (#2411) --- ngraph/python/tests/__init__.py | 24 +++++- ngraph/python/tests/runtime.py | 9 +++ ngraph/python/tests/test_ngraph/test_basic.py | 3 +- .../test_ngraph/test_sequence_processing.py | 4 +- ngraph/python/tests/test_onnx/test_backend.py | 78 +++++++++++-------- .../python/tests/test_onnx/test_zoo_models.py | 14 ++-- 6 files changed, 87 insertions(+), 45 deletions(-) diff --git a/ngraph/python/tests/__init__.py b/ngraph/python/tests/__init__.py index 4640c8e683e..34e3ecb4573 100644 --- a/ngraph/python/tests/__init__.py +++ b/ngraph/python/tests/__init__.py @@ -45,6 +45,8 @@ xfail_issue_33535 = xfail_test(reason="nGraph does not support the following ONN "DynamicQuantizeLinear") xfail_issue_33538 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" "Scan") +skip_issue_38084 = pytest.mark.skip(reason="Aborted (core dumped) Assertion " + "`(layer->get_output_partial_shape(i).is_static())' failed.") xfail_issue_33540 = xfail_test(reason="RuntimeError: GRUCell operation has a form that is not supported " "GRUCell_ should be converted to GRUCellIE operation") xfail_issue_33589 = xfail_test(reason="nGraph does not support the following ONNX operations:" @@ -85,8 +87,6 @@ xfail_issue_35921 = xfail_test(reason="ValueError - shapes mismatch in gemm") xfail_issue_35923 = xfail_test(reason="RuntimeError: PReLU without weights is not supported") xfail_issue_35924 = xfail_test(reason="Assertion error - elu results mismatch") xfail_issue_35925 = xfail_test(reason="Assertion error - reduction ops results mismatch") -xfail_issue_35926 = xfail_test(reason="RuntimeError: [NOT_IMPLEMENTED] Input image format I64 is " - "not supported yet...") xfail_issue_35927 = xfail_test(reason="RuntimeError: B has zero dimension that is not allowable") xfail_issue_35929 = xfail_test(reason="RuntimeError: Incorrect precision f64!") xfail_issue_35930 = xfail_test(reason="onnx.onnx_cpp2py_export.checker.ValidationError: " @@ -200,3 +200,23 @@ xfail_issue_37957 = xfail_test(reason="RuntimeError: nGraph does not support the "com.microsoft.CropAndResize, com.microsoft.GatherND," "com.microsoft.Pad, com.microsoft.Range") xfail_issue_39669 = xfail_test(reason="AssertionError: This model has no test data") +xfail_issue_36534 = xfail_test(reason="RuntimeError: node input index is out of range") +xfail_issue_36535 = xfail_test(reason="RuntimeError: get_shape was called on a descriptor::Tensor " + "with dynamic shape") +xfail_issue_36536 = xfail_test(reason="RuntimeError: can't protect") +xfail_issue_36537 = xfail_test(reason="ngraph.exceptions.UserInputError: (Provided tensor's shape: " + " does not match the expected: ") +xfail_issue_36538 = xfail_test(reason="RuntimeError: Check 'PartialShape::broadcast_merge_into( pshape, " + "node->get_input_partial_shape(i), autob)' failed at " + "/openvino/ngraph/src/ngraph/op/util/elementwise_args.cpp:48:") +xfail_issue_39656 = xfail_test(reason="RuntimeError: Reshape reshaped has dynamic second input!") +xfail_issue_39658 = xfail_test(reason="RuntimeError: Tile operation has a form that is not supported." + " z should be converted to TileIE operation.") +xfail_issue_39659 = xfail_test(reason="RuntimeError: Broadcast operation has a form that is not supported." + " y should be converted to Tile operation.") +xfail_issue_39661 = xfail_test(reason="RuntimeError: NonMaxSuppression operation has a form that is not " + "supported. selected_indices should be converted to " + "NonMaxSuppressionIE operation.") +xfail_issue_39662 = xfail_test(reason="RuntimeError: 'ScatterElementsUpdate' layer with name 'y' have " + "indices value that points to non-existing output tensor element") +xfail_issue_39663 = xfail_test(reason="RuntimeError: Unsupported primitive of type: ROIAlign name: Y") diff --git a/ngraph/python/tests/runtime.py b/ngraph/python/tests/runtime.py index 3afb6ebd11f..d2bff4398ef 100644 --- a/ngraph/python/tests/runtime.py +++ b/ngraph/python/tests/runtime.py @@ -23,6 +23,7 @@ from openvino.inference_engine import IECore, IENetwork from ngraph.exceptions import UserInputError from ngraph.impl import Function, Node, PartialShape from ngraph.utils.types import NumericData, get_shape + import tests log = logging.getLogger(__name__) @@ -38,6 +39,12 @@ def get_runtime(): return runtime(backend_name=tests.BACKEND_NAME) +def convert_i64_to_i32(cnn_network: IENetwork) -> None: + for cnn_input in cnn_network.input_info: + if cnn_network.input_info[cnn_input].precision == "I64": + cnn_network.input_info[cnn_input].precision = "I32" + + class Runtime(object): """Represents an nGraph runtime environment.""" @@ -98,6 +105,8 @@ class Computation(object): cnn_network = IENetwork(capsule) if self.function.is_dynamic(): cnn_network.reshape(dict(zip(param_names, input_shapes))) + # Convert inputs of the network from I64 to I32 + convert_i64_to_i32(cnn_network) self.network_cache[str(input_shapes)] = cnn_network else: cnn_network = self.network_cache[str(input_shapes)] diff --git a/ngraph/python/tests/test_ngraph/test_basic.py b/ngraph/python/tests/test_ngraph/test_basic.py index 35f62fbfc4d..5baa6f11012 100644 --- a/ngraph/python/tests/test_ngraph/test_basic.py +++ b/ngraph/python/tests/test_ngraph/test_basic.py @@ -28,7 +28,6 @@ from tests.runtime import get_runtime from tests.test_ngraph.util import run_op_node from tests import (xfail_issue_34323, xfail_issue_35929, - xfail_issue_35926, xfail_issue_36476, xfail_issue_36478, xfail_issue_36479, @@ -69,7 +68,7 @@ def test_ngraph_function_api(): pytest.param(np.int8, marks=xfail_issue_36479), np.int16, np.int32, - pytest.param(np.int64, marks=xfail_issue_35926), + np.int64, pytest.param(np.uint8, marks=xfail_issue_36479), np.uint16, pytest.param(np.uint32, marks=xfail_issue_36476), diff --git a/ngraph/python/tests/test_ngraph/test_sequence_processing.py b/ngraph/python/tests/test_ngraph/test_sequence_processing.py index 4a7a698fca1..2c9c5d25b63 100644 --- a/ngraph/python/tests/test_ngraph/test_sequence_processing.py +++ b/ngraph/python/tests/test_ngraph/test_sequence_processing.py @@ -18,7 +18,7 @@ import numpy as np import ngraph as ng from tests.runtime import get_runtime from tests.test_ngraph.util import run_op_node -from tests import xfail_issue_36478, xfail_issue_35926 +from tests import xfail_issue_36478, skip_issue_38084 def test_onehot(): @@ -46,7 +46,7 @@ def test_one_hot(): assert np.allclose(result, excepted) -@xfail_issue_35926 +@skip_issue_38084 def test_range(): start = 5 stop = 35 diff --git a/ngraph/python/tests/test_onnx/test_backend.py b/ngraph/python/tests/test_onnx/test_backend.py index bdd021e58a8..6da6151195b 100644 --- a/ngraph/python/tests/test_onnx/test_backend.py +++ b/ngraph/python/tests/test_onnx/test_backend.py @@ -25,12 +25,17 @@ import onnx.backend.test from tests.test_onnx.utils.onnx_backend import OpenVinoTestBackend from tests import (BACKEND_NAME, + skip_issue_38084, + xfail_issue_36535, + xfail_issue_39656, + xfail_issue_39658, + xfail_issue_39659, + xfail_issue_39661, + xfail_issue_39662, + xfail_issue_39663, xfail_issue_33540, xfail_issue_34314, - xfail_issue_35926, - xfail_issue_38084, xfail_issue_33616, - xfail_issue_38085, xfail_issue_38086, xfail_issue_38087, xfail_issue_35923, @@ -127,6 +132,21 @@ OnnxBackendPyTorchConvertedModelTest = None globals().update(backend_test.enable_report().test_cases) tests_expected_to_fail = [ + (skip_issue_38084, + "OnnxBackendNodeModelTest.test_expand_dim_changed_cpu", + "OnnxBackendNodeModelTest.test_expand_dim_unchanged_cpu", + "OnnxBackendSimpleModelTest.test_expand_shape_model1_cpu", + "OnnxBackendSimpleModelTest.test_expand_shape_model2_cpu", + "OnnxBackendSimpleModelTest.test_expand_shape_model3_cpu", + "OnnxBackendSimpleModelTest.test_expand_shape_model4_cpu", + "OnnxBackendNodeModelTest.test_slice_default_axes_cpu", + "OnnxBackendNodeModelTest.test_top_k_cpu", + "OnnxBackendNodeModelTest.test_top_k_negative_axis_cpu", + "OnnxBackendNodeModelTest.test_top_k_smallest_cpu", + "OnnxBackendNodeModelTest.test_nonzero_example_cpu", + "OnnxBackendNodeModelTest.test_range_int32_type_negative_delta_cpu", + "OnnxBackendNodeModelTest.test_range_float_type_positive_delta_cpu", + "OnnxBackendNodeModelTest.test_upsample_nearest_cpu"), (xfail_issue_34314, "OnnxBackendNodeModelTest.test_rnn_seq_length_cpu", "OnnxBackendNodeModelTest.test_simple_rnn_defaults_cpu", @@ -135,13 +155,11 @@ tests_expected_to_fail = [ "OnnxBackendNodeModelTest.test_gru_defaults_cpu", "OnnxBackendNodeModelTest.test_gru_seq_length_cpu", "OnnxBackendNodeModelTest.test_gru_with_initial_bias_cpu"), - (xfail_issue_35926, - "OnnxBackendNodeModelTest.test_expand_dim_changed_cpu", - "OnnxBackendNodeModelTest.test_expand_dim_unchanged_cpu", - "OnnxBackendSimpleModelTest.test_expand_shape_model1_cpu", - "OnnxBackendSimpleModelTest.test_expand_shape_model2_cpu", - "OnnxBackendSimpleModelTest.test_expand_shape_model3_cpu", - "OnnxBackendSimpleModelTest.test_expand_shape_model4_cpu", + (xfail_issue_36535, + "OnnxBackendNodeModelTest.test_constant_pad_cpu", + "OnnxBackendNodeModelTest.test_edge_pad_cpu", + "OnnxBackendNodeModelTest.test_reflect_pad_cpu"), + (xfail_issue_39656, "OnnxBackendNodeModelTest.test_reshape_extended_dims_cpu", "OnnxBackendNodeModelTest.test_reshape_negative_dim_cpu", "OnnxBackendNodeModelTest.test_reshape_one_dim_cpu", @@ -150,13 +168,15 @@ tests_expected_to_fail = [ "OnnxBackendNodeModelTest.test_reshape_reordered_all_dims_cpu", "OnnxBackendNodeModelTest.test_reshape_reordered_last_dims_cpu", "OnnxBackendNodeModelTest.test_reshape_zero_and_negative_dim_cpu", - "OnnxBackendNodeModelTest.test_reshape_zero_dim_cpu", + "OnnxBackendNodeModelTest.test_reshape_zero_dim_cpu"), + (xfail_issue_39658, "OnnxBackendNodeModelTest.test_tile_cpu", - "OnnxBackendNodeModelTest.test_tile_precomputed_cpu", + "OnnxBackendNodeModelTest.test_tile_precomputed_cpu"), + (xfail_issue_39659, "OnnxBackendNodeModelTest.test_constantofshape_float_ones_cpu", "OnnxBackendNodeModelTest.test_constantofshape_int_zeros_cpu", - "OnnxBackendNodeModelTest.test_scatter_with_axis_cpu", - "OnnxBackendNodeModelTest.test_scatter_without_axis_cpu", + "OnnxBackendNodeModelTest.test_constantofshape_int_shape_zero_cpu"), + (xfail_issue_39661, "OnnxBackendNodeModelTest.test_nonmaxsuppression_center_point_box_format_cpu", "OnnxBackendNodeModelTest.test_nonmaxsuppression_flipped_coordinates_cpu", "OnnxBackendNodeModelTest.test_nonmaxsuppression_identical_boxes_cpu", @@ -165,12 +185,14 @@ tests_expected_to_fail = [ "OnnxBackendNodeModelTest.test_nonmaxsuppression_suppress_by_IOU_and_scores_cpu", "OnnxBackendNodeModelTest.test_nonmaxsuppression_suppress_by_IOU_cpu", "OnnxBackendNodeModelTest.test_nonmaxsuppression_two_batches_cpu", + "OnnxBackendNodeModelTest.test_nonmaxsuppression_two_classes_cpu"), + (xfail_issue_39663, + "OnnxBackendNodeModelTest.test_roialign_cpu"), + (xfail_issue_39662, "OnnxBackendNodeModelTest.test_nonmaxsuppression_two_classes_cpu", "OnnxBackendNodeModelTest.test_slice_default_axes_cpu", "OnnxBackendNodeModelTest.test_roialign_cpu", - "OnnxBackendNodeModelTest.test_scatter_elements_with_axis_cpu", "OnnxBackendNodeModelTest.test_scatter_elements_with_negative_indices_cpu", - "OnnxBackendNodeModelTest.test_scatter_elements_without_axis_cpu", "OnnxBackendNodeModelTest.test_constant_pad_cpu", "OnnxBackendNodeModelTest.test_edge_pad_cpu", "OnnxBackendNodeModelTest.test_reflect_pad_cpu", @@ -178,34 +200,19 @@ tests_expected_to_fail = [ "OnnxBackendNodeModelTest.test_top_k_negative_axis_cpu", "OnnxBackendNodeModelTest.test_top_k_smallest_cpu", "OnnxBackendNodeModelTest.test_where_long_example_cpu", - "OnnxBackendNodeModelTest.test_gather_0_cpu", - "OnnxBackendNodeModelTest.test_gather_1_cpu", "OnnxBackendNodeModelTest.test_mod_int64_fmod_cpu", - "OnnxBackendNodeModelTest.test_reversesequence_batch_cpu", - "OnnxBackendNodeModelTest.test_reversesequence_time_cpu", "OnnxBackendPyTorchOperatorModelTest.test_operator_non_float_params_cpu", - "OnnxBackendPyTorchConvertedModelTest.test_Embedding_cpu", - "OnnxBackendPyTorchConvertedModelTest.test_Embedding_sparse_cpu", "OnnxBackendNodeModelTest.test_constantofshape_int_shape_zero_cpu", "OnnxBackendNodeModelTest.test_max_int64_cpu", - "OnnxBackendNodeModelTest.test_pow_types_float32_int64_cpu", "OnnxBackendNodeModelTest.test_pow_types_float_cpu", "OnnxBackendNodeModelTest.test_pow_types_int64_float32_cpu", "OnnxBackendNodeModelTest.test_pow_types_int64_int64_cpu", - "OnnxBackendNodeModelTest.test_pow_types_int_cpu", "OnnxBackendNodeModelTest.test_min_int64_cpu", - "OnnxBackendNodeModelTest.test_gather_negative_indices_cpu", - "OnnxBackendNodeModelTest.test_scatternd_cpu"), - (xfail_issue_38084, - "OnnxBackendNodeModelTest.test_nonzero_example_cpu", - "OnnxBackendNodeModelTest.test_range_int32_type_negative_delta_cpu", - "OnnxBackendNodeModelTest.test_range_float_type_positive_delta_cpu"), + "OnnxBackendNodeModelTest.test_gather_negative_indices_cpu"), (xfail_issue_33616, "OnnxBackendNodeModelTest.test_maxpool_2d_ceil_cpu", "OnnxBackendNodeModelTest.test_maxpool_2d_dilations_cpu", "OnnxBackendNodeModelTest.test_averagepool_2d_ceil_cpu"), - (xfail_issue_38085, - "OnnxBackendNodeModelTest.test_upsample_nearest_cpu"), (xfail_issue_38086, "OnnxBackendNodeModelTest.test_dynamicquantizelinear_min_adjusted_expanded_cpu", "OnnxBackendNodeModelTest.test_dynamicquantizelinear_expanded_cpu", @@ -284,6 +291,12 @@ tests_expected_to_fail = [ (xfail_issue_38089, "OnnxBackendPyTorchConvertedModelTest.test_GLU_dim_cpu"), (xfail_issue_38090, + "OnnxBackendNodeModelTest.test_where_long_example_cpu", + "OnnxBackendNodeModelTest.test_mod_int64_fmod_cpu", + "OnnxBackendPyTorchOperatorModelTest.test_operator_non_float_params_cpu", + "OnnxBackendNodeModelTest.test_max_int64_cpu", + "OnnxBackendNodeModelTest.test_pow_types_int64_int64_cpu", + "OnnxBackendNodeModelTest.test_min_int64_cpu", "OnnxBackendNodeModelTest.test_not_2d_cpu", "OnnxBackendNodeModelTest.test_not_3d_cpu", "OnnxBackendNodeModelTest.test_not_4d_cpu", @@ -342,6 +355,7 @@ tests_expected_to_fail = [ "OnnxBackendNodeModelTest.test_clip_default_int8_min_cpu", "OnnxBackendNodeModelTest.test_clip_default_int8_max_cpu"), (xfail_issue_38091, + "OnnxBackendNodeModelTest.test_gather_negative_indices_cpu", "OnnxBackendNodeModelTest.test_round_cpu", "OnnxBackendNodeModelTest.test_mvn_cpu", "OnnxBackendNodeModelTest.test_elu_example_cpu"), diff --git a/ngraph/python/tests/test_onnx/test_zoo_models.py b/ngraph/python/tests/test_onnx/test_zoo_models.py index 1134f60d2b7..4e596e30ba3 100644 --- a/ngraph/python/tests/test_onnx/test_zoo_models.py +++ b/ngraph/python/tests/test_onnx/test_zoo_models.py @@ -30,7 +30,7 @@ from tests import ( xfail_issue_39683, xfail_issue_36533, xfail_issue_39684, - xfail_issue_35926, + xfail_issue_34323, xfail_issue_36537, xfail_issue_39685, xfail_issue_37957, @@ -139,11 +139,11 @@ if len(zoo_models) > 0: (xfail_issue_36533, "test_onnx_model_zoo_vision_object_detection_segmentation_duc_model_ResNet101_DUC_7_ResNet101_DUC_HDC_ResNet101_DUC_HDC_cpu"), (xfail_issue_36533, "test_onnx_model_zoo_vision_object_detection_segmentation_retinanet_model_retinanet_9_test_retinanet_resnet101_retinanet_9_cpu"), (xfail_issue_39684, "test_onnx_model_zoo_vision_object_detection_segmentation_yolov4_model_yolov4_yolov4_yolov4_cpu"), - (xfail_issue_35926, "test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_10_download_sample_10_bertsquad10_cpu"), - (xfail_issue_35926, "test_onnx_model_zoo_text_machine_comprehension_gpt_2_model_gpt2_10_GPT2_model_cpu"), - (xfail_issue_35926, "test_onnx_model_zoo_text_machine_comprehension_roberta_model_roberta_base_11_roberta_base_11_roberta_base_11_cpu"), - (xfail_issue_35926, "test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_8_download_sample_8_bertsquad8_cpu"), - (xfail_issue_35926, "test_onnx_model_zoo_text_machine_comprehension_gpt_2_model_gpt2_lm_head_10_GPT_2_LM_HEAD_model_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_text_machine_comprehension_gpt_2_model_gpt2_10_GPT2_model_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_text_machine_comprehension_gpt_2_model_gpt2_lm_head_10_GPT_2_LM_HEAD_model_cpu"), + (xfail_issue_34323, "test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_10_download_sample_10_bertsquad10_cpu"), + (xfail_issue_34323, "test_onnx_model_zoo_text_machine_comprehension_roberta_model_roberta_base_11_roberta_base_11_roberta_base_11_cpu"), + (xfail_issue_34323, "test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_8_download_sample_8_bertsquad8_cpu"), (xfail_issue_36537, "test_onnx_model_zoo_vision_classification_efficientnet_lite4_model_efficientnet_lite4_11_efficientnet_lite4_efficientnet_lite4_cpu"), (xfail_issue_39685, "test_onnx_model_zoo_text_machine_comprehension_roberta_model_roberta_sequence_classification_9_roberta_sequence_classification_9_roberta_sequence_classification_9_cpu"), (xfail_issue_39669, "test_onnx_model_zoo_text_machine_comprehension_t5_model_t5_encoder_12_t5_encoder_cpu"), @@ -186,7 +186,7 @@ if len(zoo_models) > 0: (xfail_issue_38084, "test_MSFT_opset10_faster_rcnn_faster_rcnn_R_50_FPN_1x_cpu"), (xfail_issue_39669, "test_MSFT_opset9_cgan_cgan_cpu"), - (xfail_issue_35926, "test_MSFT_opset10_BERT_Squad_bertsquad10_cpu"), + (xfail_issue_34323, "test_MSFT_opset10_BERT_Squad_bertsquad10_cpu"), ] for test_case in import_xfail_list + execution_xfail_list: xfail, test_name = test_case