diff --git a/.ci/openvino-onnx/Jenkinsfile b/.ci/openvino-onnx/Jenkinsfile index ea78d89def2..3ae1ca8eec9 100644 --- a/.ci/openvino-onnx/Jenkinsfile +++ b/.ci/openvino-onnx/Jenkinsfile @@ -69,7 +69,7 @@ def buildDockerImage() { def runTests() { sh """ docker run --name ${DOCKER_CONTAINER_NAME} \ - --volume ${HOME}/ONNX_CI/models/.onnx:/root/.onnx ${DOCKER_IMAGE_TAG} + --volume ${HOME}/ONNX_CI/onnx-models/.onnx:/root/.onnx ${DOCKER_IMAGE_TAG} """ } diff --git a/ngraph/python/tests/__init__.py b/ngraph/python/tests/__init__.py index cb3c175395a..4c78909e3bb 100644 --- a/ngraph/python/tests/__init__.py +++ b/ngraph/python/tests/__init__.py @@ -35,9 +35,32 @@ def xfail_test(reason="Mark the test as expected to fail", strict=True): skip_segfault = pytest.mark.skip(reason="Segmentation fault error") +xfail_issue_33488 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" + "MaxUnpool") +xfail_issue_33512 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" + "Einsum") +xfail_issue_33515 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" + "BitShift") +xfail_issue_33535 = xfail_test(reason="nGraph does not support the following ONNX operations:" + "DynamicQuantizeLinear") +xfail_issue_33538 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" + "Scan") xfail_issue_33540 = xfail_test(reason="RuntimeError: GRUCell operation has a form that is not supported " "GRUCell_ should be converted to GRUCellIE operation") +xfail_issue_33589 = xfail_test(reason="nGraph does not support the following ONNX operations:" + "IsNaN and isInf") +xfail_issue_33595 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" + "Unique") +xfail_issue_33596 = xfail_test(reason="RuntimeError: nGraph does not support different sequence operations:" + "ConcatFromSequence, SequenceConstruct, SequenceAt, SplitToSequence," + "SequenceEmpty, SequenceInsert, SequenceErase, SequenceLength ") +xfail_issue_33606 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" + "Det") xfail_issue_33616 = xfail_test(reason="Add ceil_mode for Max and Avg pooling (reference implementation)") +xfail_issue_33644 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" + "Compress") +xfail_issue_33651 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" + "TfIdfVectorizer") xfail_issue_34310 = xfail_test(reason="RuntimeError: Error of validate layer: LSTMSequence_ with " "type: LSTMSequence. Layer is not instance of RNNLayer class") xfail_issue_34314 = xfail_test(reason="RuntimeError: RNNCell operation has a form that is not " @@ -45,6 +68,8 @@ xfail_issue_34314 = xfail_test(reason="RuntimeError: RNNCell operation has a for xfail_issue_34323 = xfail_test(reason="RuntimeError: data [value] doesn't exist") xfail_issue_34327 = xfail_test(reason="RuntimeError: '' layer has different " "IN and OUT channels number") +xfail_issue_33581 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" + "GatherElements") xfail_issue_35911 = xfail_test(reason="Assertion error: Pad model mismatch error") xfail_issue_35912 = xfail_test(reason="RuntimeError: Error of validate layer: B with type: " "Pad. Cannot parse parameter pads_end from IR for layer B. " @@ -59,7 +84,6 @@ xfail_issue_35917 = xfail_test(reason="RuntimeError: Unsupported input dims coun xfail_issue_35918 = xfail_test(reason="onnx.onnx_cpp2py_export.checker.ValidationError: " "Mismatched attribute type in 'test_node : alpha'") xfail_issue_35921 = xfail_test(reason="ValueError - shapes mismatch in gemm") - xfail_issue_35923 = xfail_test(reason="RuntimeError: PReLU without weights is not supported") xfail_issue_35924 = xfail_test(reason="Assertion error - elu results mismatch") xfail_issue_35925 = xfail_test(reason="Assertion error - reduction ops results mismatch") @@ -78,8 +102,6 @@ xfail_issue_36478 = xfail_test(reason="RuntimeError: [NOT_IMPLEMENTED] Input ima xfail_issue_36479 = xfail_test(reason="Assertion error - basic computation on ndarrays mismatch") xfail_issue_36480 = xfail_test(reason="RuntimeError: [NOT_FOUND] Unsupported property dummy_option " "by CPU plugin") -xfail_issue_36481 = xfail_test(reason="TypeError: _get_node_factory() takes from 0 to 1 positional " - "arguments but 2 were given") xfail_issue_36483 = xfail_test(reason="RuntimeError: Unsupported primitive of type: " "Ceiling name: ") xfail_issue_36485 = xfail_test(reason="RuntimeError: Check 'm_group >= 1' failed at " @@ -87,10 +109,10 @@ xfail_issue_36485 = xfail_test(reason="RuntimeError: Check 'm_group >= 1' failed xfail_issue_36486 = xfail_test(reason="RuntimeError: HardSigmoid operation should be converted " "to HardSigmoid_IE") xfail_issue_36487 = xfail_test(reason="Assertion error - mvn operator computation mismatch") -xfail_issue_38084 = xfail_test(reason="RuntimeError: AssertionFailed: layer->get_output_partial_shape(i) " - "is_static() nGraph NonZero operation with name: result cannot be" - "converted to NonZero layer with name: result because output" - "with index 0 contains dynamic shapes: {2,?}. Try to use " +xfail_issue_38084 = xfail_test(reason="RuntimeError: AssertionFailed: layer->get_output_partial_shape(i)" + "is_static() nGraph operation with name: cannot be" + "converted to layer with name: because output" + "with index 0 contains dynamic shapes: {}. Try to use " "CNNNetwork::reshape() method in order to specialize shapes " "before the conversion.") xfail_issue_38085 = xfail_test(reason="RuntimeError: Interpolate operation should be converted to Interp") @@ -105,14 +127,7 @@ xfail_issue_38090 = xfail_test(reason="AssertionError: Items types are not equal xfail_issue_38091 = xfail_test(reason="AssertionError: Mismatched elements") xfail_issue_38699 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" "ai.onnx.preview.training.Gradient") -xfail_issue_38700 = xfail_test(reason="RuntimeError: nGraph does not support different sequence operations:" - "ConcatFromSequence, SequenceConstruct, SequenceAt, SplitToSequence," - "SequenceEmpty, SequenceInsert, SequenceErase, SequenceLength ") xfail_issue_38701 = xfail_test(reason="RuntimeError: unsupported element type: STRING") -xfail_issue_38702 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" - "Unique") -xfail_issue_38704 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" - "TfIdfVectorizer") xfail_issue_38705 = xfail_test(reason="IndexError: deque::_M_range_check: __n (which is 0)" ">= this->size() (which is 0)") xfail_issue_38706 = xfail_test(reason="RuntimeError: output_3.0 has zero dimension which is not allowed") @@ -120,11 +135,7 @@ xfail_issue_38707 = xfail_test(reason="RuntimeError: nGraph does not support the "SoftmaxCrossEntropyLoss") xfail_issue_38708 = xfail_test(reason="RuntimeError: While validating ONNX node '': " "Axes input must be constant") -xfail_issue_38709 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" - "Scan") xfail_issue_38710 = xfail_test(reason="RuntimeError: roi has zero dimension which is not allowed") -xfail_issue_38711 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" - "GatherElements") xfail_issue_38712 = xfail_test(reason="RuntimeError: Check '(fmod == 1) " "While validating ONNX node '': " "Only 'fmod=1' mode is supported for mod operator.") @@ -142,15 +153,9 @@ xfail_issue_38715 = xfail_test(reason="RuntimeError: While validating ONNX node "Squeeze_13532[0]:i32{}, Squeeze_13529[0]:i32{}) -> (dynamic?)'" "with friendly_name 'OneHot_13534':" "Depth must be integral element type.") -xfail_issue_38716 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" - "MaxUnpool") xfail_issue_38717 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" "GreaterOrEqual") -xfail_issue_38718 = xfail_test(reason="nGraph does not support the following ONNX operations:" - "IsNaN and isInf") xfail_issue_38719 = xfail_test(reason="nGraph does not support the following ONNX operations: GatherND") -xfail_issue_38720 = xfail_test(reason="nGraph does not support the following ONNX operations:" - "DynamicQuantizeLinear") xfail_issue_38721 = xfail_test(reason="RuntimeError: While validating ONNX node '': " "While validating node 'v1::Power Power_" "(x[0]:f32{3}, y[0]:i64{3}) -> (dynamic?)' with friendly_name " @@ -169,14 +174,6 @@ xfail_issue_38725 = xfail_test(reason="RuntimeError: While validating ONNX node "value info has no element type specified") xfail_issue_38726 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" "LessOrEqual") -xfail_issue_38727 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" - "Einsum") -xfail_issue_38728 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" - "Det") -xfail_issue_38729 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" - "Compress") -xfail_issue_38731 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" - "BitShift") xfail_issue_38732 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" "ConvInteger") xfail_issue_38733 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" @@ -188,13 +185,25 @@ xfail_issue_38735 = xfail_test(reason="RuntimeError: nGraph does not support the xfail_issue_38736 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" "NegativeLogLikelihoodLoss") -# Model Zoo issues: -xfail_issue_36534 = xfail_test(reason="RuntimeError: node input index is out of range") -xfail_issue_36535 = xfail_test(reason="RuntimeError: get_shape was called on a descriptor::Tensor " - "with dynamic shape") -xfail_issue_36536 = xfail_test(reason="RuntimeError: can't protect") -xfail_issue_36537 = xfail_test(reason="ngraph.exceptions.UserInputError: (Provided tensor's shape: " - " does not match the expected: ") -xfail_issue_36538 = xfail_test(reason="RuntimeError: Check 'PartialShape::broadcast_merge_into( pshape, " - "node->get_input_partial_shape(i), autob)' failed at " - "/openvino/ngraph/src/ngraph/op/util/elementwise_args.cpp:48:") +# Model ONNX Zoo issues: +xfail_issue_36533 = xfail_test(reason="AssertionError: zoo models results mismatch") +xfail_issue_36537 = xfail_test(reason="ngraph.exceptions.UserInputError: (Provided tensor's shape:" + "%s does not match the expected: %s." + ">, >)") +xfail_issue_37687 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" + "Loop") +xfail_issue_39682 = xfail_test(reason="model with IR version >= 3 must specify opset_import for ONNX") +xfail_issue_39683 = xfail_test(reason="convolution.W in initializer but not in graph input") +xfail_issue_39684 = xfail_test(reason="ngraph.exceptions.UserInputError:" + "('Expected %s parameters, received %s.', 1, 3)") +xfail_issue_39685 = xfail_test(reason="RuntimeError: While validating node 'v1::Transpose 315," + "Constant_9353 -> (f32{?,?,?,?})' with friendly_name '315':" + "Input order must have shape [n], where n is the rank of arg.") + +# Model MSFT issues: +xfail_issue_36465 = xfail_test(reason="LSTM_Seq_lens: RuntimeError: get_shape was called on a " + "descriptor::Tensor with dynamic shape") +xfail_issue_37957 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" + "com.microsoft.CropAndResize, com.microsoft.GatherND," + "com.microsoft.Pad, com.microsoft.Range") +xfail_issue_39669 = xfail_test(reason="AssertionError: This model has no test data") diff --git a/ngraph/python/tests/test_onnx/model_zoo_preprocess.sh b/ngraph/python/tests/test_onnx/model_zoo_preprocess.sh index d063f603a54..25a742943e4 100644 --- a/ngraph/python/tests/test_onnx/model_zoo_preprocess.sh +++ b/ngraph/python/tests/test_onnx/model_zoo_preprocess.sh @@ -2,16 +2,20 @@ set -e MODELS_DIR=false +CLEAN_DIR=false +ENABLE_MSFT=false +CLONE=false function print_help { echo "Model preprocessing options:" echo " -h display this help message" echo " -c clone ONNX models repository" echo " -m set location of the models" - echo " -f clean target directory(during clone)" + echo " -f clean target directory(during clone or after enable MSFT models)" + echo " -e download and prepare MSFT models" } -while getopts ":hcfm:" opt; do +while getopts ":hcefm:" opt; do case ${opt} in h ) print_help @@ -31,23 +35,30 @@ while getopts ":hcfm:" opt; do f ) CLEAN_DIR=true ;; + e ) + ENABLE_MSFT=true + ;; esac done shift $((OPTIND -1)) if [ $MODELS_DIR = false ] ; then - echo "Unknown location of the ONNX ZOO models" + echo "Unknown location of the ZOO models" exit 170 fi +ONNX_MODELS_DIR=$MODELS_DIR/model_zoo/onnx_model_zoo +MSFT_MODELS_DIR=$MODELS_DIR/model_zoo/MSFT + if [ $CLONE = true ] ; then if [ $CLEAN_DIR = true ] ; then - rm -rf $MODELS_DIR + rm -rf $ONNX_MODELS_DIR fi - git clone https://github.com/onnx/models.git $MODELS_DIR + git clone https://github.com/onnx/models.git $ONNX_MODELS_DIR fi -cd $MODELS_DIR +mkdir -p $ONNX_MODELS_DIR +cd $ONNX_MODELS_DIR # remove already downloaded models git clean -f -x -d git checkout . @@ -55,11 +66,11 @@ git pull -p # pull models from the lfs repository # onnx models are included in the tar.gz archives git lfs pull --include="*" --exclude="*.onnx" -find $MODELS_DIR -name "*.onnx" | while read filename; do rm "$filename"; done; +find $ONNX_MODELS_DIR -name "*.onnx" | while read filename; do rm "$filename"; done; echo "extracting tar.gz archives..." -find $MODELS_DIR -name '*.tar.gz' -execdir sh -c 'BASEDIR=$(basename "{}" .tar.gz) && mkdir -p $BASEDIR' \; -execdir sh -c 'BASEDIR=$(basename "{}" .tar.gz) && tar -xzvf "{}" -C $BASEDIR --strip-components=1' \; +find $ONNX_MODELS_DIR -name '*.tar.gz' -execdir sh -c 'BASEDIR=$(basename "{}" .tar.gz) && mkdir -p $BASEDIR' \; -execdir sh -c 'BASEDIR=$(basename "{}" .tar.gz) && tar -xzvf "{}" -C $BASEDIR' \; # fix yolo v4 model -cd $MODELS_DIR/vision/object_detection_segmentation/yolov4/model/yolov4/yolov4/test_data_set +cd $ONNX_MODELS_DIR/vision/object_detection_segmentation/yolov4/model/yolov4/yolov4/test_data_set mv input0.pb input_0.pb mv input1.pb input_1.pb mv input2.pb input_2.pb @@ -67,6 +78,16 @@ mv output0.pb output_0.pb mv output1.pb output_1.pb mv output2.pb output_2.pb # fix roberta model -cd $MODELS_DIR/text/machine_comprehension/roberta/model/roberta-sequence-classification-9/ +cd $ONNX_MODELS_DIR/text/machine_comprehension/roberta/model/roberta-sequence-classification-9/roberta-sequence-classification-9 mkdir test_data_set_0 mv *.pb test_data_set_0/ + +# Prepare MSFT models +if [ $ENABLE_MSFT = true ] ; then + if [ $CLEAN_DIR = true ] ; then + rm -rf $MSFT_MODELS_DIR + fi + mkdir -p $MSFT_MODELS_DIR + wget -O $MSFT_MODELS_DIR/20191107.zip https://onnxruntimetestdata.blob.core.windows.net/models/20191107.zip + unzip $MSFT_MODELS_DIR/20191107.zip -d $MSFT_MODELS_DIR/20191107 && rm $MSFT_MODELS_DIR/20191107.zip +fi diff --git a/ngraph/python/tests/test_onnx/test_backend.py b/ngraph/python/tests/test_onnx/test_backend.py index 3e0cbbfb0fd..e73f70ac15f 100644 --- a/ngraph/python/tests/test_onnx/test_backend.py +++ b/ngraph/python/tests/test_onnx/test_backend.py @@ -48,37 +48,37 @@ from tests import (BACKEND_NAME, xfail_issue_38091, xfail_issue_35929, xfail_issue_38699, - xfail_issue_38700, + xfail_issue_33596, xfail_issue_38701, - xfail_issue_38702, - xfail_issue_38704, + xfail_issue_33595, + xfail_issue_33651, xfail_issue_38705, xfail_issue_38706, xfail_issue_38736, xfail_issue_38707, xfail_issue_38708, - xfail_issue_38709, + xfail_issue_33538, xfail_issue_38710, - xfail_issue_38711, + xfail_issue_33581, xfail_issue_38712, xfail_issue_38713, xfail_issue_38714, xfail_issue_38715, - xfail_issue_38716, + xfail_issue_33488, xfail_issue_38717, - xfail_issue_38718, + xfail_issue_33589, xfail_issue_38719, - xfail_issue_38720, + xfail_issue_33535, xfail_issue_38721, xfail_issue_38722, xfail_issue_38723, xfail_issue_38724, xfail_issue_38725, xfail_issue_38726, - xfail_issue_38727, - xfail_issue_38728, - xfail_issue_38729, - xfail_issue_38731, + xfail_issue_33512, + xfail_issue_33606, + xfail_issue_33644, + xfail_issue_33515, xfail_issue_38732, xfail_issue_38733, xfail_issue_38734, @@ -379,7 +379,7 @@ tests_expected_to_fail = [ (xfail_issue_38699, "OnnxBackendSimpleModelTest.test_gradient_of_add_and_mul_cpu", "OnnxBackendSimpleModelTest.test_gradient_of_add_cpu"), - (xfail_issue_38700, + (xfail_issue_33596, "OnnxBackendSimpleModelTest.test_sequence_model5_cpu", "OnnxBackendSimpleModelTest.test_sequence_model7_cpu", "OnnxBackendSimpleModelTest.test_sequence_model1_cpu", @@ -403,20 +403,20 @@ tests_expected_to_fail = [ "OnnxBackendNodeModelTest.test_strnormalizer_export_monday_casesensintive_upper_cpu", "OnnxBackendNodeModelTest.test_cast_STRING_to_FLOAT_cpu", "OnnxBackendNodeModelTest.test_cast_FLOAT_to_STRING_cpu"), - (xfail_issue_38702, + (xfail_issue_33595, "OnnxBackendNodeModelTest.test_unique_not_sorted_without_axis_cpu", "OnnxBackendNodeModelTest.test_unique_sorted_with_negative_axis_cpu", "OnnxBackendNodeModelTest.test_unique_sorted_with_axis_cpu", "OnnxBackendNodeModelTest.test_unique_sorted_with_axis_3d_cpu", "OnnxBackendNodeModelTest.test_unique_sorted_without_axis_cpu"), - (xfail_issue_38704, + (xfail_issue_33651, "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_batch_onlybigrams_skip5_cpu", "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_onlybigrams_levelempty_cpu", "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_batch_onlybigrams_skip0_cpu", "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_uniandbigrams_skip5_cpu", "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_only_bigrams_skip0_cpu", "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_batch_uniandbigrams_skip5_cpu", - "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_onlybigrams_skip5_cpu"), + "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_onlybigrams_skip5_cpu"), (xfail_issue_38705, "OnnxBackendNodeModelTest.test_training_dropout_mask_cpu", "OnnxBackendNodeModelTest.test_training_dropout_default_mask_cpu", @@ -523,7 +523,7 @@ tests_expected_to_fail = [ "OnnxBackendNodeModelTest.test_slice_cpu", "OnnxBackendNodeModelTest.test_slice_end_out_of_bounds_cpu", "OnnxBackendNodeModelTest.test_slice_start_out_of_bounds_cpu"), - (xfail_issue_38709, + (xfail_issue_33538, "OnnxBackendNodeModelTest.test_scan_sum_cpu", "OnnxBackendNodeModelTest.test_scan9_sum_cpu"), (xfail_issue_38710, @@ -540,28 +540,28 @@ tests_expected_to_fail = [ "OnnxBackendNodeModelTest.test_resize_upsample_scales_cubic_A_n0p5_exclude_outside_cpu", "OnnxBackendNodeModelTest.test_resize_downsample_scales_nearest_cpu", "OnnxBackendNodeModelTest.test_resize_downsample_scales_linear_cpu"), - (xfail_issue_38711, + (xfail_issue_33581, "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_no_weight_reduction_mean_ignore_index_expanded_cpu", # noqa "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1_ignore_index_expanded_cpu", # noqa "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_expanded_cpu", "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1_weight_expanded_cpu", "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded_cpu", # noqa - "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1_mean_weight_negative_ignore_index_expanded_cpu", # noqa - "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded_cpu", # noqa - "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_ignore_index_expanded_cpu", # noqa - "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1_expanded_cpu", - "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded_cpu", # noqa - "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_expanded_cpu", # noqa - "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NC_expanded_cpu", - "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_expanded_cpu", # noqa - "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_iinput_shape_is_NCd1_weight_ignore_index_expanded_cpu", # noqa - "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_mean_expanded_cpu", # noqa - "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_expanded_cpu", # noqa - "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_sum_expanded_cpu", # noqa - "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_mean_expanded_cpu", # noqa - "OnnxBackendNodeModelTest.test_gather_elements_0_cpu", - "OnnxBackendNodeModelTest.test_gather_elements_negative_indices_cpu", - "OnnxBackendNodeModelTest.test_gather_elements_1_cpu"), + "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1_mean_weight_negative_ignore_index_expanded_cpu", # noqa + "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded_cpu", # noqa + "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_ignore_index_expanded_cpu", # noqa + "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1_expanded_cpu", + "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded_cpu", # noqa + "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_expanded_cpu", # noqa + "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NC_expanded_cpu", + "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_expanded_cpu", # noqa + "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_iinput_shape_is_NCd1_weight_ignore_index_expanded_cpu", # noqa + "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_mean_expanded_cpu", # noqa + "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_expanded_cpu", # noqa + "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_sum_expanded_cpu", # noqa + "OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_mean_expanded_cpu", # noqa + "OnnxBackendNodeModelTest.test_gather_elements_0_cpu", + "OnnxBackendNodeModelTest.test_gather_elements_negative_indices_cpu", + "OnnxBackendNodeModelTest.test_gather_elements_1_cpu"), (xfail_issue_38712, "OnnxBackendNodeModelTest.test_mod_mixed_sign_int16_cpu", "OnnxBackendNodeModelTest.test_mod_uint8_cpu", @@ -591,13 +591,13 @@ tests_expected_to_fail = [ "OnnxBackendNodeModelTest.test_onehot_with_negative_axis_cpu", "OnnxBackendNodeModelTest.test_onehot_with_axis_cpu", "OnnxBackendNodeModelTest.test_onehot_negative_indices_cpu"), - (xfail_issue_38716, + (xfail_issue_33488, "OnnxBackendNodeModelTest.test_maxunpool_export_with_output_shape_cpu", "OnnxBackendNodeModelTest.test_maxunpool_export_without_output_shape_cpu"), (xfail_issue_38717, "OnnxBackendNodeModelTest.test_greater_equal_bcast_cpu", "OnnxBackendNodeModelTest.test_greater_equal_cpu"), - (xfail_issue_38718, + (xfail_issue_33589, "OnnxBackendNodeModelTest.test_isnan_cpu", "OnnxBackendNodeModelTest.test_isinf_positive_cpu", "OnnxBackendNodeModelTest.test_isinf_negative_cpu", @@ -606,7 +606,7 @@ tests_expected_to_fail = [ "OnnxBackendNodeModelTest.test_gathernd_example_int32_cpu", "OnnxBackendNodeModelTest.test_gathernd_example_float32_cpu", "OnnxBackendNodeModelTest.test_gathernd_example_int32_batch_dim1_cpu"), - (xfail_issue_38720, + (xfail_issue_33535, "OnnxBackendNodeModelTest.test_dynamicquantizelinear_min_adjusted_cpu", "OnnxBackendNodeModelTest.test_dynamicquantizelinear_cpu", "OnnxBackendNodeModelTest.test_dynamicquantizelinear_max_adjusted_cpu"), @@ -633,21 +633,21 @@ tests_expected_to_fail = [ (xfail_issue_38726, "OnnxBackendNodeModelTest.test_less_equal_cpu", "OnnxBackendNodeModelTest.test_less_equal_bcast_cpu"), - (xfail_issue_38727, + (xfail_issue_33512, "OnnxBackendNodeModelTest.test_einsum_transpose_cpu", "OnnxBackendNodeModelTest.test_einsum_batch_diagonal_cpu", "OnnxBackendNodeModelTest.test_einsum_batch_matmul_cpu", "OnnxBackendNodeModelTest.test_einsum_sum_cpu", "OnnxBackendNodeModelTest.test_einsum_inner_prod_cpu"), - (xfail_issue_38728, + (xfail_issue_33606, "OnnxBackendNodeModelTest.test_det_2d_cpu", "OnnxBackendNodeModelTest.test_det_nd_cpu"), - (xfail_issue_38729, + (xfail_issue_33644, "OnnxBackendNodeModelTest.test_compress_negative_axis_cpu", "OnnxBackendNodeModelTest.test_compress_default_axis_cpu", "OnnxBackendNodeModelTest.test_compress_1_cpu", "OnnxBackendNodeModelTest.test_compress_0_cpu"), - (xfail_issue_38731, + (xfail_issue_33515, "OnnxBackendNodeModelTest.test_bitshift_left_uint8_cpu", "OnnxBackendNodeModelTest.test_bitshift_right_uint64_cpu", "OnnxBackendNodeModelTest.test_bitshift_right_uint16_cpu", diff --git a/ngraph/python/tests/test_onnx/test_zoo_models.py b/ngraph/python/tests/test_onnx/test_zoo_models.py index a298e9f6273..d5de6f26d0a 100644 --- a/ngraph/python/tests/test_onnx/test_zoo_models.py +++ b/ngraph/python/tests/test_onnx/test_zoo_models.py @@ -23,6 +23,22 @@ import os from tests.test_onnx.utils import OpenVinoOnnxBackend from tests.test_onnx.utils.model_importer import ModelImportRunner +from tests import ( + xfail_issue_38701, + xfail_issue_39682, + xfail_issue_37687, + xfail_issue_39683, + xfail_issue_36533, + xfail_issue_39684, + xfail_issue_35926, + xfail_issue_36537, + xfail_issue_39685, + xfail_issue_37957, + xfail_issue_36465, + xfail_issue_38090, + xfail_issue_38084, + xfail_issue_39669, + xfail_issue_38726) MODELS_ROOT_DIR = tests.MODEL_ZOO_DIR @@ -93,33 +109,92 @@ if len(zoo_models) > 0: # flake8: noqa: E501 if tests.MODEL_ZOO_XFAIL: import_xfail_list = [ - "test_onnx_model_zoo_vision_classification_mnist_model_mnist_1_model_cpu", - "test_onnx_model_zoo_vision_object_detection_segmentation_tiny_yolov2_model_tinyyolov2_1_model_cpu", - "test_onnx_model_zoo_vision_object_detection_segmentation_yolov3_model_yolov3_10_yolov3_cpu", - "test_onnx_model_zoo_vision_object_detection_segmentation_ssd_mobilenetv1_model_ssd_mobilenet_v1_10_ssd_mobilenet_v1_cpu", - "test_onnx_model_zoo_text_machine_comprehension_bidirectional_attention_flow_model_bidaf_9_bidaf_cpu", + # ONNX Model Zoo + (xfail_issue_38701, "test_onnx_model_zoo_text_machine_comprehension_bidirectional_attention_flow_model_bidaf_9_bidaf_bidaf_cpu"), + (xfail_issue_39682, "test_onnx_model_zoo_vision_classification_mnist_model_mnist_1_mnist_model_cpu"), + (xfail_issue_37687, "test_onnx_model_zoo_vision_object_detection_segmentation_ssd_mobilenetv1_model_ssd_mobilenet_v1_10_ssd_mobilenet_v1_ssd_mobilenet_v1_cpu"), + (xfail_issue_37687, "test_onnx_model_zoo_vision_object_detection_segmentation_yolov3_model_yolov3_10_yolov3_yolov3_cpu"), + (xfail_issue_37687, "test_onnx_model_zoo_vision_object_detection_segmentation_tiny_yolov3_model_tiny_yolov3_11_yolov3_tiny_cpu"), + (xfail_issue_39683, "test_onnx_model_zoo_vision_object_detection_segmentation_tiny_yolov2_model_tinyyolov2_1_tiny_yolov2_model_cpu"), + (xfail_issue_38726, "test_onnx_model_zoo_text_machine_comprehension_t5_model_t5_decoder_with_lm_head_12_t5_decoder_with_lm_head_cpu"), + + # Model MSFT + (xfail_issue_37687, "test_MSFT_opset10_mlperf_ssd_mobilenet_300_ssd_mobilenet_v1_coco_2018_01_28_cpu"), + (xfail_issue_37687, "test_MSFT_opset10_mlperf_ssd_resnet34_1200_ssd_resnet34_mAP_20.2_cpu"), + (xfail_issue_37687, "test_MSFT_opset10_yolov3_yolov3_cpu"), + (xfail_issue_37687, "test_MSFT_opset11_tinyyolov3_yolov3_tiny_cpu"), + (xfail_issue_37687, "test_MSFT_opset10_mlperf_ssd_resnet34_1200_ssd_resnet34_mAP_20.2_cpu"), + (xfail_issue_37957, "test_MSFT_opset10_mask_rcnn_keras_mask_rcnn_keras_cpu"), + (xfail_issue_36465, "test_MSFT_opset9_LSTM_Seq_lens_unpacked_model_cpu"), ] for test_case in import_xfail_list: - pytest.mark.xfail(getattr(test_cases, test_case)) + xfail, test_name = test_case + xfail(getattr(test_cases, test_name)) + del test_cases test_cases = backend_test.test_cases["OnnxBackendModelExecutionTest"] if tests.MODEL_ZOO_XFAIL: execution_xfail_list = [ - "test_onnx_model_zoo_text_machine_comprehension_gpt_2_model_gpt2_10_GPT2_model_cpu", - "test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_10_bertsquad10_cpu", - "test_onnx_model_zoo_text_machine_comprehension_roberta_model_roberta_sequence_classification_9_roberta_sequence_classification_9_cpu", - "test_onnx_model_zoo_text_machine_comprehension_gpt_2_model_gpt2_lm_head_10_model_cpu", - "test_onnx_model_zoo_vision_classification_efficientnet_lite4_model_efficientnet_lite4_11_efficientnet_lite4_cpu", - "test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_8_bertsquad8_cpu", - "test_onnx_model_zoo_vision_object_detection_segmentation_duc_model_ResNet101_DUC_7_ResNet101_DUC_HDC_cpu", - "test_onnx_model_zoo_vision_object_detection_segmentation_retinanet_model_retinanet_9_retinanet_9_cpu", - "test_onnx_model_zoo_vision_classification_shufflenet_model_shufflenet_v2_10_test_shufflenetv2_model_cpu", - "test_onnx_model_zoo_vision_object_detection_segmentation_yolov4_model_yolov4_yolov4_yolov4_cpu", - "test_onnx_model_zoo_text_machine_comprehension_roberta_model_roberta_base_11_roberta_base_11_cpu", + # ONNX Model Zoo + (xfail_issue_36533, "test_onnx_model_zoo_vision_object_detection_segmentation_duc_model_ResNet101_DUC_7_ResNet101_DUC_HDC_ResNet101_DUC_HDC_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_vision_object_detection_segmentation_retinanet_model_retinanet_9_test_retinanet_resnet101_retinanet_9_cpu"), + (xfail_issue_39684, "test_onnx_model_zoo_vision_object_detection_segmentation_yolov4_model_yolov4_yolov4_yolov4_cpu"), + (xfail_issue_35926, "test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_10_download_sample_10_bertsquad10_cpu"), + (xfail_issue_35926, "test_onnx_model_zoo_text_machine_comprehension_gpt_2_model_gpt2_10_GPT2_model_cpu"), + (xfail_issue_35926, "test_onnx_model_zoo_text_machine_comprehension_roberta_model_roberta_base_11_roberta_base_11_roberta_base_11_cpu"), + (xfail_issue_35926, "test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_8_download_sample_8_bertsquad8_cpu"), + (xfail_issue_35926, "test_onnx_model_zoo_text_machine_comprehension_gpt_2_model_gpt2_lm_head_10_GPT_2_LM_HEAD_model_cpu"), + (xfail_issue_36537, "test_onnx_model_zoo_vision_classification_shufflenet_model_shufflenet_v2_10_model_test_shufflenetv2_model_cpu"), + (xfail_issue_36537, "test_onnx_model_zoo_vision_classification_efficientnet_lite4_model_efficientnet_lite4_11_efficientnet_lite4_efficientnet_lite4_cpu"), + (xfail_issue_39685, "test_onnx_model_zoo_text_machine_comprehension_roberta_model_roberta_sequence_classification_9_roberta_sequence_classification_9_roberta_sequence_classification_9_cpu"), + (xfail_issue_39669, "test_onnx_model_zoo_text_machine_comprehension_t5_model_t5_encoder_12_t5_encoder_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_vision_classification_mobilenet_model_mobilenetv2_7_mobilenetv2_1.0_mobilenetv2_1.0_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_vision_classification_resnet_model_resnet101_v2_7_resnet101v2_resnet101_v2_7_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_vision_classification_resnet_model_resnet34_v2_7_resnet34v2_resnet34_v2_7_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_vision_classification_vgg_model_vgg16_7_vgg16_vgg16_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_vision_classification_resnet_model_resnet152_v2_7_resnet152v2_resnet152_v2_7_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_vision_classification_vgg_model_vgg19_bn_7_vgg19_bn_vgg19_bn_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_vision_object_detection_segmentation_tiny_yolov2_model_tinyyolov2_7_tiny_yolov2_model_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_vision_object_detection_segmentation_tiny_yolov2_model_tinyyolov2_8_tiny_yolov2_Model_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_vision_style_transfer_fast_neural_style_model_mosaic_8_mosaic_mosaic_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_vision_classification_resnet_model_resnet18_v2_7_resnet18v2_resnet18_v2_7_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_vision_classification_resnet_model_resnet101_v1_7_resnet101v1_resnet101_v1_7_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_vision_classification_resnet_model_resnet152_v1_7_resnet152v1_resnet152_v1_7_cpu"), + (xfail_issue_36533, "test_onnx_model_zoo_vision_classification_densenet_121_model_densenet_3_densenet121_model_cpu"), + (xfail_issue_38084, "test_onnx_model_zoo_vision_object_detection_segmentation_mask_rcnn_model_MaskRCNN_10_mask_rcnn_R_50_FPN_1x_cpu"), + (xfail_issue_38090, "test_onnx_model_zoo_vision_object_detection_segmentation_ssd_model_ssd_10_model_cpu"), + (xfail_issue_38084, "test_onnx_model_zoo_vision_object_detection_segmentation_faster_rcnn_model_FasterRCNN_10_faster_rcnn_R_50_FPN_1x_cpu"), + + # Model MSFT + (xfail_issue_36533, "test_MSFT_opset10_tf_inception_v2_model_cpu"), + (xfail_issue_36533, "test_MSFT_opset8_test_tiny_yolov2_model_cpu"), + (xfail_issue_36533, "test_MSFT_opset9_tf_inception_v2_model_cpu"), + (xfail_issue_36533, "test_MSFT_opset8_tf_inception_v2_model_cpu"), + (xfail_issue_36533, "test_MSFT_opset7_test_resnet152v2_resnet152v2_cpu"), + (xfail_issue_36533, "test_MSFT_opset7_test_tiny_yolov2_model_cpu"), + (xfail_issue_36533, "test_MSFT_opset7_tf_inception_v2_model_cpu"), + (xfail_issue_36533, "test_MSFT_opset11_tf_inception_v2_model_cpu"), + (xfail_issue_36533, "test_MSFT_opset7_test_mobilenetv2_1.0_mobilenetv2_1.0_cpu"), + + (xfail_issue_38090, "test_MSFT_opset7_fp16_tiny_yolov2_onnxzoo_winmlperf_tiny_yolov2_cpu"), + (xfail_issue_38090, "test_MSFT_opset8_fp16_inception_v1_onnxzoo_lotus_inception_v1_cpu"), + (xfail_issue_38090, "test_MSFT_opset8_fp16_tiny_yolov2_onnxzoo_winmlperf_tiny_yolov2_cpu"), + (xfail_issue_38090, "test_MSFT_opset8_fp16_shufflenet_onnxzoo_lotus_shufflenet_cpu"), + (xfail_issue_38090, "test_MSFT_opset7_fp16_inception_v1_onnxzoo_lotus_inception_v1_cpu"), + (xfail_issue_38090, "test_MSFT_opset10_mlperf_resnet_resnet50_v1_cpu"), + (xfail_issue_38090, "test_MSFT_opset7_fp16_shufflenet_onnxzoo_lotus_shufflenet_cpu"), + + (xfail_issue_38084, "test_MSFT_opset10_mask_rcnn_mask_rcnn_R_50_FPN_1x_cpu"), + (xfail_issue_38084, "test_MSFT_opset10_faster_rcnn_faster_rcnn_R_50_FPN_1x_cpu"), + + (xfail_issue_39669, "test_MSFT_opset9_cgan_cgan_cpu"), + (xfail_issue_35926, "test_MSFT_opset10_BERT_Squad_bertsquad10_cpu"), ] for test_case in import_xfail_list + execution_xfail_list: - pytest.mark.xfail(getattr(test_cases, test_case)) + xfail, test_name = test_case + xfail(getattr(test_cases, test_name)) + del test_cases globals().update(backend_test.enable_report().test_cases) diff --git a/ngraph/python/tests/test_onnx/utils/model_importer.py b/ngraph/python/tests/test_onnx/utils/model_importer.py index 5c5ab86b9a2..afda808f35c 100644 --- a/ngraph/python/tests/test_onnx/utils/model_importer.py +++ b/ngraph/python/tests/test_onnx/utils/model_importer.py @@ -145,5 +145,5 @@ class ModelImportRunner(onnx.backend.test.BackendTest): model_test.model_dir, prepared_model, model_test.rtol, model_test.atol ) - assert executed_tests > 0, "This model have no test data" + assert executed_tests > 0, "This model has no test data" self._add_test("ModelExecution", model_test.name, run_execution, model_marker) diff --git a/ngraph/python/tox.ini b/ngraph/python/tox.ini index 8ccb754f9cf..d586faf7c31 100644 --- a/ngraph/python/tox.ini +++ b/ngraph/python/tox.ini @@ -27,7 +27,8 @@ commands= flake8 {posargs:src/ setup.py} flake8 --ignore=D100,D101,D102,D103,D104,D105,D107,W503 tests/ # ignore lack of docs in tests mypy --config-file=tox.ini {posargs:src/} - pytest --backend={env:NGRAPH_BACKEND} tests -v -n 20 -k 'not _cuda' --model_zoo_xfail + pytest --backend={env:NGRAPH_BACKEND} tests -v -k 'not _cuda' --ignore=tests/test_onnx/test_zoo_models.py + pytest --backend={env:NGRAPH_BACKEND} tests/test_onnx/test_zoo_models.py -v -n 10 -k 'not _cuda' --model_zoo_xfail [testenv:devenv] envdir = devenv