Add MSFT models to OpenVino ONNX CI check (#2377)
* Add MSFT models to OpenVino ONNX CI and update xfails * Update paths * Fix typo and update xfails * Unset xfails to check current behavior * Update: * add MSFT models to preprocesing script * update xfail names * Final update of xfail test cases * Update xfail paths * Update xfails * Uncomment part of preprocesing script * Update script * Add cleaning support for MSFT preprocesing * Add -e option to script help * Initilze variables * Update ngraph/python/tests/test_onnx/model_zoo_preprocess.sh Fix a mistake Co-authored-by: Tomasz Socha <tomasz.socha@intel.com> * Update ngraph/python/tests/test_onnx/model_zoo_preprocess.sh align spaces Co-authored-by: Tomasz Socha <tomasz.socha@intel.com> Co-authored-by: Tomasz Socha <tomasz.socha@intel.com>
This commit is contained in:
Rafal Blaczkowski
GitHub
parent
1cfa47cde8
commit
e4875f2a6b
2
.ci/openvino-onnx/Jenkinsfile
vendored
2
.ci/openvino-onnx/Jenkinsfile
vendored
@ -69,7 +69,7 @@ def buildDockerImage() {
|
||||
def runTests() {
|
||||
sh """
|
||||
docker run --name ${DOCKER_CONTAINER_NAME} \
|
||||
--volume ${HOME}/ONNX_CI/models/.onnx:/root/.onnx ${DOCKER_IMAGE_TAG}
|
||||
--volume ${HOME}/ONNX_CI/onnx-models/.onnx:/root/.onnx ${DOCKER_IMAGE_TAG}
|
||||
"""
|
||||
}
|
||||
|
||||
|
||||
@ -35,9 +35,32 @@ def xfail_test(reason="Mark the test as expected to fail", strict=True):
|
||||
|
||||
|
||||
skip_segfault = pytest.mark.skip(reason="Segmentation fault error")
|
||||
xfail_issue_33488 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"MaxUnpool")
|
||||
xfail_issue_33512 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"Einsum")
|
||||
xfail_issue_33515 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"BitShift")
|
||||
xfail_issue_33535 = xfail_test(reason="nGraph does not support the following ONNX operations:"
|
||||
"DynamicQuantizeLinear")
|
||||
xfail_issue_33538 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"Scan")
|
||||
xfail_issue_33540 = xfail_test(reason="RuntimeError: GRUCell operation has a form that is not supported "
|
||||
"GRUCell_<number> should be converted to GRUCellIE operation")
|
||||
xfail_issue_33589 = xfail_test(reason="nGraph does not support the following ONNX operations:"
|
||||
"IsNaN and isInf")
|
||||
xfail_issue_33595 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"Unique")
|
||||
xfail_issue_33596 = xfail_test(reason="RuntimeError: nGraph does not support different sequence operations:"
|
||||
"ConcatFromSequence, SequenceConstruct, SequenceAt, SplitToSequence,"
|
||||
"SequenceEmpty, SequenceInsert, SequenceErase, SequenceLength ")
|
||||
xfail_issue_33606 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"Det")
|
||||
xfail_issue_33616 = xfail_test(reason="Add ceil_mode for Max and Avg pooling (reference implementation)")
|
||||
xfail_issue_33644 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"Compress")
|
||||
xfail_issue_33651 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"TfIdfVectorizer")
|
||||
xfail_issue_34310 = xfail_test(reason="RuntimeError: Error of validate layer: LSTMSequence_<number> with "
|
||||
"type: LSTMSequence. Layer is not instance of RNNLayer class")
|
||||
xfail_issue_34314 = xfail_test(reason="RuntimeError: RNNCell operation has a form that is not "
|
||||
@ -45,6 +68,8 @@ xfail_issue_34314 = xfail_test(reason="RuntimeError: RNNCell operation has a for
|
||||
xfail_issue_34323 = xfail_test(reason="RuntimeError: data [value] doesn't exist")
|
||||
xfail_issue_34327 = xfail_test(reason="RuntimeError: '<value>' layer has different "
|
||||
"IN and OUT channels number")
|
||||
xfail_issue_33581 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"GatherElements")
|
||||
xfail_issue_35911 = xfail_test(reason="Assertion error: Pad model mismatch error")
|
||||
xfail_issue_35912 = xfail_test(reason="RuntimeError: Error of validate layer: B with type: "
|
||||
"Pad. Cannot parse parameter pads_end from IR for layer B. "
|
||||
@ -59,7 +84,6 @@ xfail_issue_35917 = xfail_test(reason="RuntimeError: Unsupported input dims coun
|
||||
xfail_issue_35918 = xfail_test(reason="onnx.onnx_cpp2py_export.checker.ValidationError: "
|
||||
"Mismatched attribute type in 'test_node : alpha'")
|
||||
xfail_issue_35921 = xfail_test(reason="ValueError - shapes mismatch in gemm")
|
||||
|
||||
xfail_issue_35923 = xfail_test(reason="RuntimeError: PReLU without weights is not supported")
|
||||
xfail_issue_35924 = xfail_test(reason="Assertion error - elu results mismatch")
|
||||
xfail_issue_35925 = xfail_test(reason="Assertion error - reduction ops results mismatch")
|
||||
@ -78,8 +102,6 @@ xfail_issue_36478 = xfail_test(reason="RuntimeError: [NOT_IMPLEMENTED] Input ima
|
||||
xfail_issue_36479 = xfail_test(reason="Assertion error - basic computation on ndarrays mismatch")
|
||||
xfail_issue_36480 = xfail_test(reason="RuntimeError: [NOT_FOUND] Unsupported property dummy_option "
|
||||
"by CPU plugin")
|
||||
xfail_issue_36481 = xfail_test(reason="TypeError: _get_node_factory() takes from 0 to 1 positional "
|
||||
"arguments but 2 were given")
|
||||
xfail_issue_36483 = xfail_test(reason="RuntimeError: Unsupported primitive of type: "
|
||||
"Ceiling name: <value>")
|
||||
xfail_issue_36485 = xfail_test(reason="RuntimeError: Check 'm_group >= 1' failed at "
|
||||
@ -87,10 +109,10 @@ xfail_issue_36485 = xfail_test(reason="RuntimeError: Check 'm_group >= 1' failed
|
||||
xfail_issue_36486 = xfail_test(reason="RuntimeError: HardSigmoid operation should be converted "
|
||||
"to HardSigmoid_IE")
|
||||
xfail_issue_36487 = xfail_test(reason="Assertion error - mvn operator computation mismatch")
|
||||
xfail_issue_38084 = xfail_test(reason="RuntimeError: AssertionFailed: layer->get_output_partial_shape(i) "
|
||||
"is_static() nGraph NonZero operation with name: result cannot be"
|
||||
"converted to NonZero layer with name: result because output"
|
||||
"with index 0 contains dynamic shapes: {2,?}. Try to use "
|
||||
xfail_issue_38084 = xfail_test(reason="RuntimeError: AssertionFailed: layer->get_output_partial_shape(i)"
|
||||
"is_static() nGraph <value> operation with name: <value> cannot be"
|
||||
"converted to <value> layer with name: <value> because output"
|
||||
"with index 0 contains dynamic shapes: {<value>}. Try to use "
|
||||
"CNNNetwork::reshape() method in order to specialize shapes "
|
||||
"before the conversion.")
|
||||
xfail_issue_38085 = xfail_test(reason="RuntimeError: Interpolate operation should be converted to Interp")
|
||||
@ -105,14 +127,7 @@ xfail_issue_38090 = xfail_test(reason="AssertionError: Items types are not equal
|
||||
xfail_issue_38091 = xfail_test(reason="AssertionError: Mismatched elements")
|
||||
xfail_issue_38699 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"ai.onnx.preview.training.Gradient")
|
||||
xfail_issue_38700 = xfail_test(reason="RuntimeError: nGraph does not support different sequence operations:"
|
||||
"ConcatFromSequence, SequenceConstruct, SequenceAt, SplitToSequence,"
|
||||
"SequenceEmpty, SequenceInsert, SequenceErase, SequenceLength ")
|
||||
xfail_issue_38701 = xfail_test(reason="RuntimeError: unsupported element type: STRING")
|
||||
xfail_issue_38702 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"Unique")
|
||||
xfail_issue_38704 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"TfIdfVectorizer")
|
||||
xfail_issue_38705 = xfail_test(reason="IndexError: deque::_M_range_check: __n (which is 0)"
|
||||
">= this->size() (which is 0)")
|
||||
xfail_issue_38706 = xfail_test(reason="RuntimeError: output_3.0 has zero dimension which is not allowed")
|
||||
@ -120,11 +135,7 @@ xfail_issue_38707 = xfail_test(reason="RuntimeError: nGraph does not support the
|
||||
"SoftmaxCrossEntropyLoss")
|
||||
xfail_issue_38708 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(Slice): y>': "
|
||||
"Axes input must be constant")
|
||||
xfail_issue_38709 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"Scan")
|
||||
xfail_issue_38710 = xfail_test(reason="RuntimeError: roi has zero dimension which is not allowed")
|
||||
xfail_issue_38711 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"GatherElements")
|
||||
xfail_issue_38712 = xfail_test(reason="RuntimeError: Check '(fmod == 1) "
|
||||
"While validating ONNX node '<Node(Mod): z>': "
|
||||
"Only 'fmod=1' mode is supported for mod operator.")
|
||||
@ -142,15 +153,9 @@ xfail_issue_38715 = xfail_test(reason="RuntimeError: While validating ONNX node
|
||||
"Squeeze_13532[0]:i32{}, Squeeze_13529[0]:i32{}) -> (dynamic?)'"
|
||||
"with friendly_name 'OneHot_13534':"
|
||||
"Depth must be integral element type.")
|
||||
xfail_issue_38716 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"MaxUnpool")
|
||||
xfail_issue_38717 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"GreaterOrEqual")
|
||||
xfail_issue_38718 = xfail_test(reason="nGraph does not support the following ONNX operations:"
|
||||
"IsNaN and isInf")
|
||||
xfail_issue_38719 = xfail_test(reason="nGraph does not support the following ONNX operations: GatherND")
|
||||
xfail_issue_38720 = xfail_test(reason="nGraph does not support the following ONNX operations:"
|
||||
"DynamicQuantizeLinear")
|
||||
xfail_issue_38721 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(Pow): z>': "
|
||||
"While validating node 'v1::Power Power_<number>"
|
||||
"(x[0]:f32{3}, y[0]:i64{3}) -> (dynamic?)' with friendly_name "
|
||||
@ -169,14 +174,6 @@ xfail_issue_38725 = xfail_test(reason="RuntimeError: While validating ONNX node
|
||||
"value info has no element type specified")
|
||||
xfail_issue_38726 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"LessOrEqual")
|
||||
xfail_issue_38727 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"Einsum")
|
||||
xfail_issue_38728 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"Det")
|
||||
xfail_issue_38729 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"Compress")
|
||||
xfail_issue_38731 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"BitShift")
|
||||
xfail_issue_38732 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"ConvInteger")
|
||||
xfail_issue_38733 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
@ -188,13 +185,25 @@ xfail_issue_38735 = xfail_test(reason="RuntimeError: nGraph does not support the
|
||||
xfail_issue_38736 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"NegativeLogLikelihoodLoss")
|
||||
|
||||
# Model Zoo issues:
|
||||
xfail_issue_36534 = xfail_test(reason="RuntimeError: node input index is out of range")
|
||||
xfail_issue_36535 = xfail_test(reason="RuntimeError: get_shape was called on a descriptor::Tensor "
|
||||
"with dynamic shape")
|
||||
xfail_issue_36536 = xfail_test(reason="RuntimeError: can't protect")
|
||||
xfail_issue_36537 = xfail_test(reason="ngraph.exceptions.UserInputError: (Provided tensor's shape: "
|
||||
"<value> does not match the expected: <value>")
|
||||
xfail_issue_36538 = xfail_test(reason="RuntimeError: Check 'PartialShape::broadcast_merge_into( pshape, "
|
||||
"node->get_input_partial_shape(i), autob)' failed at "
|
||||
"/openvino/ngraph/src/ngraph/op/util/elementwise_args.cpp:48:")
|
||||
# Model ONNX Zoo issues:
|
||||
xfail_issue_36533 = xfail_test(reason="AssertionError: zoo models results mismatch")
|
||||
xfail_issue_36537 = xfail_test(reason="ngraph.exceptions.UserInputError: (Provided tensor's shape:"
|
||||
"%s does not match the expected: %s."
|
||||
"<PartialShape: <value>>, <PartialShape: <value>>)")
|
||||
xfail_issue_37687 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"Loop")
|
||||
xfail_issue_39682 = xfail_test(reason="model with IR version >= 3 must specify opset_import for ONNX")
|
||||
xfail_issue_39683 = xfail_test(reason="convolution.W in initializer but not in graph input")
|
||||
xfail_issue_39684 = xfail_test(reason="ngraph.exceptions.UserInputError:"
|
||||
"('Expected %s parameters, received %s.', 1, 3)")
|
||||
xfail_issue_39685 = xfail_test(reason="RuntimeError: While validating node 'v1::Transpose 315,"
|
||||
"Constant_9353 -> (f32{?,?,?,?})' with friendly_name '315':"
|
||||
"Input order must have shape [n], where n is the rank of arg.")
|
||||
|
||||
# Model MSFT issues:
|
||||
xfail_issue_36465 = xfail_test(reason="LSTM_Seq_lens: RuntimeError: get_shape was called on a "
|
||||
"descriptor::Tensor with dynamic shape")
|
||||
xfail_issue_37957 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
|
||||
"com.microsoft.CropAndResize, com.microsoft.GatherND,"
|
||||
"com.microsoft.Pad, com.microsoft.Range")
|
||||
xfail_issue_39669 = xfail_test(reason="AssertionError: This model has no test data")
|
||||
|
||||
@ -2,16 +2,20 @@
|
||||
set -e
|
||||
|
||||
MODELS_DIR=false
|
||||
CLEAN_DIR=false
|
||||
ENABLE_MSFT=false
|
||||
CLONE=false
|
||||
|
||||
function print_help {
|
||||
echo "Model preprocessing options:"
|
||||
echo " -h display this help message"
|
||||
echo " -c clone ONNX models repository"
|
||||
echo " -m <DIR> set location of the models"
|
||||
echo " -f clean target directory(during clone)"
|
||||
echo " -f clean target directory(during clone or after enable MSFT models)"
|
||||
echo " -e download and prepare MSFT models"
|
||||
}
|
||||
|
||||
while getopts ":hcfm:" opt; do
|
||||
while getopts ":hcefm:" opt; do
|
||||
case ${opt} in
|
||||
h )
|
||||
print_help
|
||||
@ -31,23 +35,30 @@ while getopts ":hcfm:" opt; do
|
||||
f )
|
||||
CLEAN_DIR=true
|
||||
;;
|
||||
e )
|
||||
ENABLE_MSFT=true
|
||||
;;
|
||||
esac
|
||||
done
|
||||
shift $((OPTIND -1))
|
||||
|
||||
if [ $MODELS_DIR = false ] ; then
|
||||
echo "Unknown location of the ONNX ZOO models"
|
||||
echo "Unknown location of the ZOO models"
|
||||
exit 170
|
||||
fi
|
||||
|
||||
ONNX_MODELS_DIR=$MODELS_DIR/model_zoo/onnx_model_zoo
|
||||
MSFT_MODELS_DIR=$MODELS_DIR/model_zoo/MSFT
|
||||
|
||||
if [ $CLONE = true ] ; then
|
||||
if [ $CLEAN_DIR = true ] ; then
|
||||
rm -rf $MODELS_DIR
|
||||
rm -rf $ONNX_MODELS_DIR
|
||||
fi
|
||||
git clone https://github.com/onnx/models.git $MODELS_DIR
|
||||
git clone https://github.com/onnx/models.git $ONNX_MODELS_DIR
|
||||
fi
|
||||
|
||||
cd $MODELS_DIR
|
||||
mkdir -p $ONNX_MODELS_DIR
|
||||
cd $ONNX_MODELS_DIR
|
||||
# remove already downloaded models
|
||||
git clean -f -x -d
|
||||
git checkout .
|
||||
@ -55,11 +66,11 @@ git pull -p
|
||||
# pull models from the lfs repository
|
||||
# onnx models are included in the tar.gz archives
|
||||
git lfs pull --include="*" --exclude="*.onnx"
|
||||
find $MODELS_DIR -name "*.onnx" | while read filename; do rm "$filename"; done;
|
||||
find $ONNX_MODELS_DIR -name "*.onnx" | while read filename; do rm "$filename"; done;
|
||||
echo "extracting tar.gz archives..."
|
||||
find $MODELS_DIR -name '*.tar.gz' -execdir sh -c 'BASEDIR=$(basename "{}" .tar.gz) && mkdir -p $BASEDIR' \; -execdir sh -c 'BASEDIR=$(basename "{}" .tar.gz) && tar -xzvf "{}" -C $BASEDIR --strip-components=1' \;
|
||||
find $ONNX_MODELS_DIR -name '*.tar.gz' -execdir sh -c 'BASEDIR=$(basename "{}" .tar.gz) && mkdir -p $BASEDIR' \; -execdir sh -c 'BASEDIR=$(basename "{}" .tar.gz) && tar -xzvf "{}" -C $BASEDIR' \;
|
||||
# fix yolo v4 model
|
||||
cd $MODELS_DIR/vision/object_detection_segmentation/yolov4/model/yolov4/yolov4/test_data_set
|
||||
cd $ONNX_MODELS_DIR/vision/object_detection_segmentation/yolov4/model/yolov4/yolov4/test_data_set
|
||||
mv input0.pb input_0.pb
|
||||
mv input1.pb input_1.pb
|
||||
mv input2.pb input_2.pb
|
||||
@ -67,6 +78,16 @@ mv output0.pb output_0.pb
|
||||
mv output1.pb output_1.pb
|
||||
mv output2.pb output_2.pb
|
||||
# fix roberta model
|
||||
cd $MODELS_DIR/text/machine_comprehension/roberta/model/roberta-sequence-classification-9/
|
||||
cd $ONNX_MODELS_DIR/text/machine_comprehension/roberta/model/roberta-sequence-classification-9/roberta-sequence-classification-9
|
||||
mkdir test_data_set_0
|
||||
mv *.pb test_data_set_0/
|
||||
|
||||
# Prepare MSFT models
|
||||
if [ $ENABLE_MSFT = true ] ; then
|
||||
if [ $CLEAN_DIR = true ] ; then
|
||||
rm -rf $MSFT_MODELS_DIR
|
||||
fi
|
||||
mkdir -p $MSFT_MODELS_DIR
|
||||
wget -O $MSFT_MODELS_DIR/20191107.zip https://onnxruntimetestdata.blob.core.windows.net/models/20191107.zip
|
||||
unzip $MSFT_MODELS_DIR/20191107.zip -d $MSFT_MODELS_DIR/20191107 && rm $MSFT_MODELS_DIR/20191107.zip
|
||||
fi
|
||||
|
||||
@ -48,37 +48,37 @@ from tests import (BACKEND_NAME,
|
||||
xfail_issue_38091,
|
||||
xfail_issue_35929,
|
||||
xfail_issue_38699,
|
||||
xfail_issue_38700,
|
||||
xfail_issue_33596,
|
||||
xfail_issue_38701,
|
||||
xfail_issue_38702,
|
||||
xfail_issue_38704,
|
||||
xfail_issue_33595,
|
||||
xfail_issue_33651,
|
||||
xfail_issue_38705,
|
||||
xfail_issue_38706,
|
||||
xfail_issue_38736,
|
||||
xfail_issue_38707,
|
||||
xfail_issue_38708,
|
||||
xfail_issue_38709,
|
||||
xfail_issue_33538,
|
||||
xfail_issue_38710,
|
||||
xfail_issue_38711,
|
||||
xfail_issue_33581,
|
||||
xfail_issue_38712,
|
||||
xfail_issue_38713,
|
||||
xfail_issue_38714,
|
||||
xfail_issue_38715,
|
||||
xfail_issue_38716,
|
||||
xfail_issue_33488,
|
||||
xfail_issue_38717,
|
||||
xfail_issue_38718,
|
||||
xfail_issue_33589,
|
||||
xfail_issue_38719,
|
||||
xfail_issue_38720,
|
||||
xfail_issue_33535,
|
||||
xfail_issue_38721,
|
||||
xfail_issue_38722,
|
||||
xfail_issue_38723,
|
||||
xfail_issue_38724,
|
||||
xfail_issue_38725,
|
||||
xfail_issue_38726,
|
||||
xfail_issue_38727,
|
||||
xfail_issue_38728,
|
||||
xfail_issue_38729,
|
||||
xfail_issue_38731,
|
||||
xfail_issue_33512,
|
||||
xfail_issue_33606,
|
||||
xfail_issue_33644,
|
||||
xfail_issue_33515,
|
||||
xfail_issue_38732,
|
||||
xfail_issue_38733,
|
||||
xfail_issue_38734,
|
||||
@ -379,7 +379,7 @@ tests_expected_to_fail = [
|
||||
(xfail_issue_38699,
|
||||
"OnnxBackendSimpleModelTest.test_gradient_of_add_and_mul_cpu",
|
||||
"OnnxBackendSimpleModelTest.test_gradient_of_add_cpu"),
|
||||
(xfail_issue_38700,
|
||||
(xfail_issue_33596,
|
||||
"OnnxBackendSimpleModelTest.test_sequence_model5_cpu",
|
||||
"OnnxBackendSimpleModelTest.test_sequence_model7_cpu",
|
||||
"OnnxBackendSimpleModelTest.test_sequence_model1_cpu",
|
||||
@ -403,20 +403,20 @@ tests_expected_to_fail = [
|
||||
"OnnxBackendNodeModelTest.test_strnormalizer_export_monday_casesensintive_upper_cpu",
|
||||
"OnnxBackendNodeModelTest.test_cast_STRING_to_FLOAT_cpu",
|
||||
"OnnxBackendNodeModelTest.test_cast_FLOAT_to_STRING_cpu"),
|
||||
(xfail_issue_38702,
|
||||
(xfail_issue_33595,
|
||||
"OnnxBackendNodeModelTest.test_unique_not_sorted_without_axis_cpu",
|
||||
"OnnxBackendNodeModelTest.test_unique_sorted_with_negative_axis_cpu",
|
||||
"OnnxBackendNodeModelTest.test_unique_sorted_with_axis_cpu",
|
||||
"OnnxBackendNodeModelTest.test_unique_sorted_with_axis_3d_cpu",
|
||||
"OnnxBackendNodeModelTest.test_unique_sorted_without_axis_cpu"),
|
||||
(xfail_issue_38704,
|
||||
(xfail_issue_33651,
|
||||
"OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_batch_onlybigrams_skip5_cpu",
|
||||
"OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_onlybigrams_levelempty_cpu",
|
||||
"OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_batch_onlybigrams_skip0_cpu",
|
||||
"OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_uniandbigrams_skip5_cpu",
|
||||
"OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_only_bigrams_skip0_cpu",
|
||||
"OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_batch_uniandbigrams_skip5_cpu",
|
||||
"OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_onlybigrams_skip5_cpu"),
|
||||
"OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_onlybigrams_skip5_cpu"),
|
||||
(xfail_issue_38705,
|
||||
"OnnxBackendNodeModelTest.test_training_dropout_mask_cpu",
|
||||
"OnnxBackendNodeModelTest.test_training_dropout_default_mask_cpu",
|
||||
@ -523,7 +523,7 @@ tests_expected_to_fail = [
|
||||
"OnnxBackendNodeModelTest.test_slice_cpu",
|
||||
"OnnxBackendNodeModelTest.test_slice_end_out_of_bounds_cpu",
|
||||
"OnnxBackendNodeModelTest.test_slice_start_out_of_bounds_cpu"),
|
||||
(xfail_issue_38709,
|
||||
(xfail_issue_33538,
|
||||
"OnnxBackendNodeModelTest.test_scan_sum_cpu",
|
||||
"OnnxBackendNodeModelTest.test_scan9_sum_cpu"),
|
||||
(xfail_issue_38710,
|
||||
@ -540,28 +540,28 @@ tests_expected_to_fail = [
|
||||
"OnnxBackendNodeModelTest.test_resize_upsample_scales_cubic_A_n0p5_exclude_outside_cpu",
|
||||
"OnnxBackendNodeModelTest.test_resize_downsample_scales_nearest_cpu",
|
||||
"OnnxBackendNodeModelTest.test_resize_downsample_scales_linear_cpu"),
|
||||
(xfail_issue_38711,
|
||||
(xfail_issue_33581,
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_no_weight_reduction_mean_ignore_index_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1_ignore_index_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_expanded_cpu",
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1_weight_expanded_cpu",
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1_mean_weight_negative_ignore_index_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_ignore_index_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1_expanded_cpu",
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NC_expanded_cpu",
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_iinput_shape_is_NCd1_weight_ignore_index_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_mean_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_sum_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_mean_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_gather_elements_0_cpu",
|
||||
"OnnxBackendNodeModelTest.test_gather_elements_negative_indices_cpu",
|
||||
"OnnxBackendNodeModelTest.test_gather_elements_1_cpu"),
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1_mean_weight_negative_ignore_index_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_ignore_index_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1_expanded_cpu",
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NC_expanded_cpu",
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_iinput_shape_is_NCd1_weight_ignore_index_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_mean_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_sum_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_mean_expanded_cpu", # noqa
|
||||
"OnnxBackendNodeModelTest.test_gather_elements_0_cpu",
|
||||
"OnnxBackendNodeModelTest.test_gather_elements_negative_indices_cpu",
|
||||
"OnnxBackendNodeModelTest.test_gather_elements_1_cpu"),
|
||||
(xfail_issue_38712,
|
||||
"OnnxBackendNodeModelTest.test_mod_mixed_sign_int16_cpu",
|
||||
"OnnxBackendNodeModelTest.test_mod_uint8_cpu",
|
||||
@ -591,13 +591,13 @@ tests_expected_to_fail = [
|
||||
"OnnxBackendNodeModelTest.test_onehot_with_negative_axis_cpu",
|
||||
"OnnxBackendNodeModelTest.test_onehot_with_axis_cpu",
|
||||
"OnnxBackendNodeModelTest.test_onehot_negative_indices_cpu"),
|
||||
(xfail_issue_38716,
|
||||
(xfail_issue_33488,
|
||||
"OnnxBackendNodeModelTest.test_maxunpool_export_with_output_shape_cpu",
|
||||
"OnnxBackendNodeModelTest.test_maxunpool_export_without_output_shape_cpu"),
|
||||
(xfail_issue_38717,
|
||||
"OnnxBackendNodeModelTest.test_greater_equal_bcast_cpu",
|
||||
"OnnxBackendNodeModelTest.test_greater_equal_cpu"),
|
||||
(xfail_issue_38718,
|
||||
(xfail_issue_33589,
|
||||
"OnnxBackendNodeModelTest.test_isnan_cpu",
|
||||
"OnnxBackendNodeModelTest.test_isinf_positive_cpu",
|
||||
"OnnxBackendNodeModelTest.test_isinf_negative_cpu",
|
||||
@ -606,7 +606,7 @@ tests_expected_to_fail = [
|
||||
"OnnxBackendNodeModelTest.test_gathernd_example_int32_cpu",
|
||||
"OnnxBackendNodeModelTest.test_gathernd_example_float32_cpu",
|
||||
"OnnxBackendNodeModelTest.test_gathernd_example_int32_batch_dim1_cpu"),
|
||||
(xfail_issue_38720,
|
||||
(xfail_issue_33535,
|
||||
"OnnxBackendNodeModelTest.test_dynamicquantizelinear_min_adjusted_cpu",
|
||||
"OnnxBackendNodeModelTest.test_dynamicquantizelinear_cpu",
|
||||
"OnnxBackendNodeModelTest.test_dynamicquantizelinear_max_adjusted_cpu"),
|
||||
@ -633,21 +633,21 @@ tests_expected_to_fail = [
|
||||
(xfail_issue_38726,
|
||||
"OnnxBackendNodeModelTest.test_less_equal_cpu",
|
||||
"OnnxBackendNodeModelTest.test_less_equal_bcast_cpu"),
|
||||
(xfail_issue_38727,
|
||||
(xfail_issue_33512,
|
||||
"OnnxBackendNodeModelTest.test_einsum_transpose_cpu",
|
||||
"OnnxBackendNodeModelTest.test_einsum_batch_diagonal_cpu",
|
||||
"OnnxBackendNodeModelTest.test_einsum_batch_matmul_cpu",
|
||||
"OnnxBackendNodeModelTest.test_einsum_sum_cpu",
|
||||
"OnnxBackendNodeModelTest.test_einsum_inner_prod_cpu"),
|
||||
(xfail_issue_38728,
|
||||
(xfail_issue_33606,
|
||||
"OnnxBackendNodeModelTest.test_det_2d_cpu",
|
||||
"OnnxBackendNodeModelTest.test_det_nd_cpu"),
|
||||
(xfail_issue_38729,
|
||||
(xfail_issue_33644,
|
||||
"OnnxBackendNodeModelTest.test_compress_negative_axis_cpu",
|
||||
"OnnxBackendNodeModelTest.test_compress_default_axis_cpu",
|
||||
"OnnxBackendNodeModelTest.test_compress_1_cpu",
|
||||
"OnnxBackendNodeModelTest.test_compress_0_cpu"),
|
||||
(xfail_issue_38731,
|
||||
(xfail_issue_33515,
|
||||
"OnnxBackendNodeModelTest.test_bitshift_left_uint8_cpu",
|
||||
"OnnxBackendNodeModelTest.test_bitshift_right_uint64_cpu",
|
||||
"OnnxBackendNodeModelTest.test_bitshift_right_uint16_cpu",
|
||||
|
||||
@ -23,6 +23,22 @@ import os
|
||||
from tests.test_onnx.utils import OpenVinoOnnxBackend
|
||||
from tests.test_onnx.utils.model_importer import ModelImportRunner
|
||||
|
||||
from tests import (
|
||||
xfail_issue_38701,
|
||||
xfail_issue_39682,
|
||||
xfail_issue_37687,
|
||||
xfail_issue_39683,
|
||||
xfail_issue_36533,
|
||||
xfail_issue_39684,
|
||||
xfail_issue_35926,
|
||||
xfail_issue_36537,
|
||||
xfail_issue_39685,
|
||||
xfail_issue_37957,
|
||||
xfail_issue_36465,
|
||||
xfail_issue_38090,
|
||||
xfail_issue_38084,
|
||||
xfail_issue_39669,
|
||||
xfail_issue_38726)
|
||||
|
||||
MODELS_ROOT_DIR = tests.MODEL_ZOO_DIR
|
||||
|
||||
@ -93,33 +109,92 @@ if len(zoo_models) > 0:
|
||||
# flake8: noqa: E501
|
||||
if tests.MODEL_ZOO_XFAIL:
|
||||
import_xfail_list = [
|
||||
"test_onnx_model_zoo_vision_classification_mnist_model_mnist_1_model_cpu",
|
||||
"test_onnx_model_zoo_vision_object_detection_segmentation_tiny_yolov2_model_tinyyolov2_1_model_cpu",
|
||||
"test_onnx_model_zoo_vision_object_detection_segmentation_yolov3_model_yolov3_10_yolov3_cpu",
|
||||
"test_onnx_model_zoo_vision_object_detection_segmentation_ssd_mobilenetv1_model_ssd_mobilenet_v1_10_ssd_mobilenet_v1_cpu",
|
||||
"test_onnx_model_zoo_text_machine_comprehension_bidirectional_attention_flow_model_bidaf_9_bidaf_cpu",
|
||||
# ONNX Model Zoo
|
||||
(xfail_issue_38701, "test_onnx_model_zoo_text_machine_comprehension_bidirectional_attention_flow_model_bidaf_9_bidaf_bidaf_cpu"),
|
||||
(xfail_issue_39682, "test_onnx_model_zoo_vision_classification_mnist_model_mnist_1_mnist_model_cpu"),
|
||||
(xfail_issue_37687, "test_onnx_model_zoo_vision_object_detection_segmentation_ssd_mobilenetv1_model_ssd_mobilenet_v1_10_ssd_mobilenet_v1_ssd_mobilenet_v1_cpu"),
|
||||
(xfail_issue_37687, "test_onnx_model_zoo_vision_object_detection_segmentation_yolov3_model_yolov3_10_yolov3_yolov3_cpu"),
|
||||
(xfail_issue_37687, "test_onnx_model_zoo_vision_object_detection_segmentation_tiny_yolov3_model_tiny_yolov3_11_yolov3_tiny_cpu"),
|
||||
(xfail_issue_39683, "test_onnx_model_zoo_vision_object_detection_segmentation_tiny_yolov2_model_tinyyolov2_1_tiny_yolov2_model_cpu"),
|
||||
(xfail_issue_38726, "test_onnx_model_zoo_text_machine_comprehension_t5_model_t5_decoder_with_lm_head_12_t5_decoder_with_lm_head_cpu"),
|
||||
|
||||
# Model MSFT
|
||||
(xfail_issue_37687, "test_MSFT_opset10_mlperf_ssd_mobilenet_300_ssd_mobilenet_v1_coco_2018_01_28_cpu"),
|
||||
(xfail_issue_37687, "test_MSFT_opset10_mlperf_ssd_resnet34_1200_ssd_resnet34_mAP_20.2_cpu"),
|
||||
(xfail_issue_37687, "test_MSFT_opset10_yolov3_yolov3_cpu"),
|
||||
(xfail_issue_37687, "test_MSFT_opset11_tinyyolov3_yolov3_tiny_cpu"),
|
||||
(xfail_issue_37687, "test_MSFT_opset10_mlperf_ssd_resnet34_1200_ssd_resnet34_mAP_20.2_cpu"),
|
||||
(xfail_issue_37957, "test_MSFT_opset10_mask_rcnn_keras_mask_rcnn_keras_cpu"),
|
||||
(xfail_issue_36465, "test_MSFT_opset9_LSTM_Seq_lens_unpacked_model_cpu"),
|
||||
]
|
||||
for test_case in import_xfail_list:
|
||||
pytest.mark.xfail(getattr(test_cases, test_case))
|
||||
xfail, test_name = test_case
|
||||
xfail(getattr(test_cases, test_name))
|
||||
|
||||
del test_cases
|
||||
|
||||
test_cases = backend_test.test_cases["OnnxBackendModelExecutionTest"]
|
||||
if tests.MODEL_ZOO_XFAIL:
|
||||
execution_xfail_list = [
|
||||
"test_onnx_model_zoo_text_machine_comprehension_gpt_2_model_gpt2_10_GPT2_model_cpu",
|
||||
"test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_10_bertsquad10_cpu",
|
||||
"test_onnx_model_zoo_text_machine_comprehension_roberta_model_roberta_sequence_classification_9_roberta_sequence_classification_9_cpu",
|
||||
"test_onnx_model_zoo_text_machine_comprehension_gpt_2_model_gpt2_lm_head_10_model_cpu",
|
||||
"test_onnx_model_zoo_vision_classification_efficientnet_lite4_model_efficientnet_lite4_11_efficientnet_lite4_cpu",
|
||||
"test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_8_bertsquad8_cpu",
|
||||
"test_onnx_model_zoo_vision_object_detection_segmentation_duc_model_ResNet101_DUC_7_ResNet101_DUC_HDC_cpu",
|
||||
"test_onnx_model_zoo_vision_object_detection_segmentation_retinanet_model_retinanet_9_retinanet_9_cpu",
|
||||
"test_onnx_model_zoo_vision_classification_shufflenet_model_shufflenet_v2_10_test_shufflenetv2_model_cpu",
|
||||
"test_onnx_model_zoo_vision_object_detection_segmentation_yolov4_model_yolov4_yolov4_yolov4_cpu",
|
||||
"test_onnx_model_zoo_text_machine_comprehension_roberta_model_roberta_base_11_roberta_base_11_cpu",
|
||||
# ONNX Model Zoo
|
||||
(xfail_issue_36533, "test_onnx_model_zoo_vision_object_detection_segmentation_duc_model_ResNet101_DUC_7_ResNet101_DUC_HDC_ResNet101_DUC_HDC_cpu"),
|
||||
(xfail_issue_36533, "test_onnx_model_zoo_vision_object_detection_segmentation_retinanet_model_retinanet_9_test_retinanet_resnet101_retinanet_9_cpu"),
|
||||
(xfail_issue_39684, "test_onnx_model_zoo_vision_object_detection_segmentation_yolov4_model_yolov4_yolov4_yolov4_cpu"),
|
||||
(xfail_issue_35926, "test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_10_download_sample_10_bertsquad10_cpu"),
|
||||
(xfail_issue_35926, "test_onnx_model_zoo_text_machine_comprehension_gpt_2_model_gpt2_10_GPT2_model_cpu"),
|
||||
(xfail_issue_35926, "test_onnx_model_zoo_text_machine_comprehension_roberta_model_roberta_base_11_roberta_base_11_roberta_base_11_cpu"),
|
||||
(xfail_issue_35926, "test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_8_download_sample_8_bertsquad8_cpu"),
|
||||
(xfail_issue_35926, "test_onnx_model_zoo_text_machine_comprehension_gpt_2_model_gpt2_lm_head_10_GPT_2_LM_HEAD_model_cpu"),
|
||||
(xfail_issue_36537, "test_onnx_model_zoo_vision_classification_shufflenet_model_shufflenet_v2_10_model_test_shufflenetv2_model_cpu"),
|
||||
(xfail_issue_36537, "test_onnx_model_zoo_vision_classification_efficientnet_lite4_model_efficientnet_lite4_11_efficientnet_lite4_efficientnet_lite4_cpu"),
|
||||
(xfail_issue_39685, "test_onnx_model_zoo_text_machine_comprehension_roberta_model_roberta_sequence_classification_9_roberta_sequence_classification_9_roberta_sequence_classification_9_cpu"),
|
||||
(xfail_issue_39669, "test_onnx_model_zoo_text_machine_comprehension_t5_model_t5_encoder_12_t5_encoder_cpu"),
|
||||
(xfail_issue_36533, "test_onnx_model_zoo_vision_classification_mobilenet_model_mobilenetv2_7_mobilenetv2_1.0_mobilenetv2_1.0_cpu"),
|
||||
(xfail_issue_36533, "test_onnx_model_zoo_vision_classification_resnet_model_resnet101_v2_7_resnet101v2_resnet101_v2_7_cpu"),
|
||||
(xfail_issue_36533, "test_onnx_model_zoo_vision_classification_resnet_model_resnet34_v2_7_resnet34v2_resnet34_v2_7_cpu"),
|
||||
(xfail_issue_36533, "test_onnx_model_zoo_vision_classification_vgg_model_vgg16_7_vgg16_vgg16_cpu"),
|
||||
(xfail_issue_36533, "test_onnx_model_zoo_vision_classification_resnet_model_resnet152_v2_7_resnet152v2_resnet152_v2_7_cpu"),
|
||||
(xfail_issue_36533, "test_onnx_model_zoo_vision_classification_vgg_model_vgg19_bn_7_vgg19_bn_vgg19_bn_cpu"),
|
||||
(xfail_issue_36533, "test_onnx_model_zoo_vision_object_detection_segmentation_tiny_yolov2_model_tinyyolov2_7_tiny_yolov2_model_cpu"),
|
||||
(xfail_issue_36533, "test_onnx_model_zoo_vision_object_detection_segmentation_tiny_yolov2_model_tinyyolov2_8_tiny_yolov2_Model_cpu"),
|
||||
(xfail_issue_36533, "test_onnx_model_zoo_vision_style_transfer_fast_neural_style_model_mosaic_8_mosaic_mosaic_cpu"),
|
||||
(xfail_issue_36533, "test_onnx_model_zoo_vision_classification_resnet_model_resnet18_v2_7_resnet18v2_resnet18_v2_7_cpu"),
|
||||
(xfail_issue_36533, "test_onnx_model_zoo_vision_classification_resnet_model_resnet101_v1_7_resnet101v1_resnet101_v1_7_cpu"),
|
||||
(xfail_issue_36533, "test_onnx_model_zoo_vision_classification_resnet_model_resnet152_v1_7_resnet152v1_resnet152_v1_7_cpu"),
|
||||
(xfail_issue_36533, "test_onnx_model_zoo_vision_classification_densenet_121_model_densenet_3_densenet121_model_cpu"),
|
||||
(xfail_issue_38084, "test_onnx_model_zoo_vision_object_detection_segmentation_mask_rcnn_model_MaskRCNN_10_mask_rcnn_R_50_FPN_1x_cpu"),
|
||||
(xfail_issue_38090, "test_onnx_model_zoo_vision_object_detection_segmentation_ssd_model_ssd_10_model_cpu"),
|
||||
(xfail_issue_38084, "test_onnx_model_zoo_vision_object_detection_segmentation_faster_rcnn_model_FasterRCNN_10_faster_rcnn_R_50_FPN_1x_cpu"),
|
||||
|
||||
# Model MSFT
|
||||
(xfail_issue_36533, "test_MSFT_opset10_tf_inception_v2_model_cpu"),
|
||||
(xfail_issue_36533, "test_MSFT_opset8_test_tiny_yolov2_model_cpu"),
|
||||
(xfail_issue_36533, "test_MSFT_opset9_tf_inception_v2_model_cpu"),
|
||||
(xfail_issue_36533, "test_MSFT_opset8_tf_inception_v2_model_cpu"),
|
||||
(xfail_issue_36533, "test_MSFT_opset7_test_resnet152v2_resnet152v2_cpu"),
|
||||
(xfail_issue_36533, "test_MSFT_opset7_test_tiny_yolov2_model_cpu"),
|
||||
(xfail_issue_36533, "test_MSFT_opset7_tf_inception_v2_model_cpu"),
|
||||
(xfail_issue_36533, "test_MSFT_opset11_tf_inception_v2_model_cpu"),
|
||||
(xfail_issue_36533, "test_MSFT_opset7_test_mobilenetv2_1.0_mobilenetv2_1.0_cpu"),
|
||||
|
||||
(xfail_issue_38090, "test_MSFT_opset7_fp16_tiny_yolov2_onnxzoo_winmlperf_tiny_yolov2_cpu"),
|
||||
(xfail_issue_38090, "test_MSFT_opset8_fp16_inception_v1_onnxzoo_lotus_inception_v1_cpu"),
|
||||
(xfail_issue_38090, "test_MSFT_opset8_fp16_tiny_yolov2_onnxzoo_winmlperf_tiny_yolov2_cpu"),
|
||||
(xfail_issue_38090, "test_MSFT_opset8_fp16_shufflenet_onnxzoo_lotus_shufflenet_cpu"),
|
||||
(xfail_issue_38090, "test_MSFT_opset7_fp16_inception_v1_onnxzoo_lotus_inception_v1_cpu"),
|
||||
(xfail_issue_38090, "test_MSFT_opset10_mlperf_resnet_resnet50_v1_cpu"),
|
||||
(xfail_issue_38090, "test_MSFT_opset7_fp16_shufflenet_onnxzoo_lotus_shufflenet_cpu"),
|
||||
|
||||
(xfail_issue_38084, "test_MSFT_opset10_mask_rcnn_mask_rcnn_R_50_FPN_1x_cpu"),
|
||||
(xfail_issue_38084, "test_MSFT_opset10_faster_rcnn_faster_rcnn_R_50_FPN_1x_cpu"),
|
||||
|
||||
(xfail_issue_39669, "test_MSFT_opset9_cgan_cgan_cpu"),
|
||||
(xfail_issue_35926, "test_MSFT_opset10_BERT_Squad_bertsquad10_cpu"),
|
||||
]
|
||||
for test_case in import_xfail_list + execution_xfail_list:
|
||||
pytest.mark.xfail(getattr(test_cases, test_case))
|
||||
xfail, test_name = test_case
|
||||
xfail(getattr(test_cases, test_name))
|
||||
|
||||
del test_cases
|
||||
|
||||
globals().update(backend_test.enable_report().test_cases)
|
||||
|
||||
@ -145,5 +145,5 @@ class ModelImportRunner(onnx.backend.test.BackendTest):
|
||||
model_test.model_dir, prepared_model, model_test.rtol, model_test.atol
|
||||
)
|
||||
|
||||
assert executed_tests > 0, "This model have no test data"
|
||||
assert executed_tests > 0, "This model has no test data"
|
||||
self._add_test("ModelExecution", model_test.name, run_execution, model_marker)
|
||||
|
||||
@ -27,7 +27,8 @@ commands=
|
||||
flake8 {posargs:src/ setup.py}
|
||||
flake8 --ignore=D100,D101,D102,D103,D104,D105,D107,W503 tests/ # ignore lack of docs in tests
|
||||
mypy --config-file=tox.ini {posargs:src/}
|
||||
pytest --backend={env:NGRAPH_BACKEND} tests -v -n 20 -k 'not _cuda' --model_zoo_xfail
|
||||
pytest --backend={env:NGRAPH_BACKEND} tests -v -k 'not _cuda' --ignore=tests/test_onnx/test_zoo_models.py
|
||||
pytest --backend={env:NGRAPH_BACKEND} tests/test_onnx/test_zoo_models.py -v -n 10 -k 'not _cuda' --model_zoo_xfail
|
||||
|
||||
[testenv:devenv]
|
||||
envdir = devenv
|
||||
|
||||
Loading…
Reference in New Issue
Block a user