2021-03-26 17:54:28 +03:00
|
|
|
# Copyright (C) 2018-2021 Intel Corporation
|
|
|
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
|
|
2020-08-07 08:58:57 +02:00
|
|
|
import pytest
|
2020-05-20 01:13:06 +03:00
|
|
|
|
|
|
|
|
# test.BACKEND_NAME is a configuration variable determining which
|
|
|
|
|
# nGraph backend tests will use. It's set during pytest configuration time.
|
|
|
|
|
# See `pytest_configure` hook in `conftest.py` for more details.
|
|
|
|
|
BACKEND_NAME = None
|
2020-07-29 09:34:35 +02:00
|
|
|
|
2020-09-21 13:55:02 +02:00
|
|
|
# test.MODEL_ZOO_DIR is a configuration variable providing the path
|
|
|
|
|
# to the ZOO of ONNX models to test. It's set during pytest configuration time.
|
|
|
|
|
# See `pytest_configure` hook in `conftest.py` for more
|
2020-07-29 09:34:35 +02:00
|
|
|
# details.
|
2020-09-21 13:55:02 +02:00
|
|
|
MODEL_ZOO_DIR = None
|
|
|
|
|
|
|
|
|
|
# test.MODEL_ZOO_XFAIL is a configuration variable which enable xfails for model zoo.
|
|
|
|
|
MODEL_ZOO_XFAIL = False
|
2020-08-07 08:58:57 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def xfail_test(reason="Mark the test as expected to fail", strict=True):
|
|
|
|
|
return pytest.mark.xfail(reason=reason, strict=strict)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
skip_segfault = pytest.mark.skip(reason="Segmentation fault error")
|
2021-11-03 09:11:28 +03:00
|
|
|
xfail_accuracy = xfail_test(reason="Accuracy")
|
2021-11-12 22:44:44 +03:00
|
|
|
xfail_issue_FLOAT_LIKE = xfail_test(reason="Use of bfloat16 or float16")
|
2021-11-03 09:11:28 +03:00
|
|
|
xfail_issue_69444 = xfail_test(reason="failed with accuracy issue")
|
2021-10-18 21:36:48 +02:00
|
|
|
skip_issue_67415 = pytest.mark.skip(reason="RuntimeError: Unsupported data type for when filling blob!")
|
|
|
|
|
xfail_issue_67415 = xfail_test(reason="RuntimeError: Unsupported data type for when filling blob!")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_33488 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations: "
|
2020-09-29 18:05:31 +02:00
|
|
|
"MaxUnpool")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_33538 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations: "
|
2020-09-29 18:05:31 +02:00
|
|
|
"Scan")
|
2020-10-22 14:06:23 +02:00
|
|
|
skip_issue_38084 = pytest.mark.skip(reason="Aborted (core dumped) Assertion "
|
|
|
|
|
"`(layer->get_output_partial_shape(i).is_static())' failed.")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_33589 = xfail_test(reason="nGraph does not support the following ONNX operations: "
|
2020-09-29 18:05:31 +02:00
|
|
|
"IsNaN and isInf")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_33595 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations: "
|
2020-09-29 18:05:31 +02:00
|
|
|
"Unique")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_33596 = xfail_test(reason="RuntimeError: nGraph does not support different sequence operations: "
|
|
|
|
|
"ConcatFromSequence, SequenceConstruct, SequenceAt, SplitToSequence, "
|
2020-09-29 18:05:31 +02:00
|
|
|
"SequenceEmpty, SequenceInsert, SequenceErase, SequenceLength ")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_33606 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations: "
|
2020-09-29 18:05:31 +02:00
|
|
|
"Det")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_33651 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations: "
|
2020-09-29 18:05:31 +02:00
|
|
|
"TfIdfVectorizer")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_33581 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations: "
|
2020-09-29 18:05:31 +02:00
|
|
|
"GatherElements")
|
2020-08-07 08:58:57 +02:00
|
|
|
xfail_issue_35923 = xfail_test(reason="RuntimeError: PReLU without weights is not supported")
|
|
|
|
|
xfail_issue_35927 = xfail_test(reason="RuntimeError: B has zero dimension that is not allowable")
|
|
|
|
|
xfail_issue_36486 = xfail_test(reason="RuntimeError: HardSigmoid operation should be converted "
|
|
|
|
|
"to HardSigmoid_IE")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_38084 = xfail_test(reason="RuntimeError: AssertionFailed: layer->get_output_partial_shape(i)."
|
|
|
|
|
"is_static() nGraph <value> operation with name: <value> cannot be "
|
|
|
|
|
"converted to <value> layer with name: <value> because output "
|
2020-09-29 18:05:31 +02:00
|
|
|
"with index 0 contains dynamic shapes: {<value>}. Try to use "
|
2020-09-14 13:07:47 +02:00
|
|
|
"CNNNetwork::reshape() method in order to specialize shapes "
|
|
|
|
|
"before the conversion.")
|
|
|
|
|
xfail_issue_38091 = xfail_test(reason="AssertionError: Mismatched elements")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_38699 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations: "
|
2020-09-14 13:07:47 +02:00
|
|
|
"ai.onnx.preview.training.Gradient")
|
|
|
|
|
xfail_issue_38701 = xfail_test(reason="RuntimeError: unsupported element type: STRING")
|
|
|
|
|
xfail_issue_38706 = xfail_test(reason="RuntimeError: output_3.0 has zero dimension which is not allowed")
|
|
|
|
|
xfail_issue_38708 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(Slice): y>': "
|
|
|
|
|
"Axes input must be constant")
|
2021-08-25 11:37:44 +02:00
|
|
|
xfail_issue_38710 = xfail_test(reason="RuntimeError: data has zero dimension which is not allowed")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_38713 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations: "
|
2020-09-14 13:07:47 +02:00
|
|
|
"ai.onnx.preview.training.Momentum")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_45457 = xfail_test(reason="RuntimeError: Unsupported dynamic ops: v5::Loop "
|
2020-12-21 12:32:15 +01:00
|
|
|
"Not constant termination condition body output is not supported")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_38724 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(Resize): Y>': "
|
|
|
|
|
"tf_crop_and_resize - this type of coordinate transformation mode "
|
|
|
|
|
"is not supported. Choose one of the following modes: "
|
|
|
|
|
"tf_half_pixel_for_nn, asymmetric, align_corners, pytorch_half_pixel, "
|
2020-09-14 13:07:47 +02:00
|
|
|
"half_pixel")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_38725 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(Loop): "
|
2020-09-14 13:07:47 +02:00
|
|
|
"value info has no element type specified")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_38734 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations: "
|
2020-09-14 13:07:47 +02:00
|
|
|
"ai.onnx.preview.training.Adam")
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_38735 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations: "
|
2020-09-14 13:07:47 +02:00
|
|
|
"ai.onnx.preview.training.Adagrad")
|
2021-02-15 11:14:20 +01:00
|
|
|
xfail_issue_48052 = xfail_test(reason="Dropout op is not supported in traning mode")
|
2021-01-18 10:47:40 +01:00
|
|
|
xfail_issue_45180 = xfail_test(reason="RuntimeError: Unsupported dynamic op: ReduceSum")
|
2021-08-25 11:37:44 +02:00
|
|
|
xfail_issue_44851 = xfail_test(reason="Expected: Unsupported dynamic op: Broadcast")
|
|
|
|
|
xfail_issue_44854 = xfail_test(reason="Expected: Unsupported dynamic op: VariadicSplit")
|
|
|
|
|
xfail_issue_44858 = xfail_test(reason="Expected: Unsupported dynamic op: Unsqueeze")
|
|
|
|
|
xfail_issue_44956 = xfail_test(reason="Expected: Unsupported dynamic op: Loop")
|
|
|
|
|
xfail_issue_44957 = xfail_test(reason="Expected: Unsupported dynamic op: NonZero")
|
|
|
|
|
xfail_issue_44958 = xfail_test(reason="Expected: Unsupported dynamic op: Interpolate")
|
|
|
|
|
xfail_issue_44965 = xfail_test(reason="Expected: RuntimeError: value info has no element")
|
|
|
|
|
xfail_issue_44968 = xfail_test(reason="Expected: Unsupported dynamic op: Squeeze")
|
2021-01-25 13:08:15 +01:00
|
|
|
xfail_issue_47323 = xfail_test(reason="RuntimeError: The plugin does not support FP64")
|
|
|
|
|
xfail_issue_47337 = xfail_test(reason="RuntimeError: Unsupported dynamic ops: v1::OneHot")
|
2020-12-11 11:57:50 +01:00
|
|
|
|
2020-09-29 18:05:31 +02:00
|
|
|
# Model MSFT issues:
|
2021-09-20 15:38:49 +03:00
|
|
|
xfail_issue_37957 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations: "
|
|
|
|
|
"com.microsoft.CropAndResize, com.microsoft.GatherND, "
|
2020-09-29 18:05:31 +02:00
|
|
|
"com.microsoft.Pad, com.microsoft.Range")
|
|
|
|
|
xfail_issue_39669 = xfail_test(reason="AssertionError: This model has no test data")
|
2020-10-22 14:06:23 +02:00
|
|
|
xfail_issue_36534 = xfail_test(reason="RuntimeError: node input index is out of range")
|
|
|
|
|
xfail_issue_36536 = xfail_test(reason="RuntimeError: can't protect")
|
|
|
|
|
xfail_issue_36538 = xfail_test(reason="RuntimeError: Check 'PartialShape::broadcast_merge_into( pshape, "
|
|
|
|
|
"node->get_input_partial_shape(i), autob)' failed at "
|
|
|
|
|
"/openvino/ngraph/src/ngraph/op/util/elementwise_args.cpp:48:")
|
|
|
|
|
xfail_issue_39658 = xfail_test(reason="RuntimeError: Tile operation has a form that is not supported."
|
|
|
|
|
" z should be converted to TileIE operation.")
|
|
|
|
|
xfail_issue_39662 = xfail_test(reason="RuntimeError: 'ScatterElementsUpdate' layer with name 'y' have "
|
|
|
|
|
"indices value that points to non-existing output tensor element")
|
2020-11-23 04:39:35 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
xfail_issue_37973 = xfail_test(reason="TF Inception V2 - AssertionError: zoo models results mismatch")
|
2021-01-27 10:22:16 +01:00
|
|
|
xfail_issue_47430 = xfail_test(reason="FCN ResNet models - AssertionError: zoo models results mismatch")
|
|
|
|
|
xfail_issue_47495 = xfail_test(reason="BertSquad-10 from MSFT - AssertionError: zoo models results mismatch")
|
2021-12-10 13:08:38 +03:00
|
|
|
xfail_issue_49207 = xfail_test(reason="Model references undeclared parameters")
|
2021-02-19 09:38:20 +03:00
|
|
|
xfail_issue_48145 = xfail_test(reason="BertSquad-8 - AssertionError: Items are not equal: ACTUAL: 4 "
|
|
|
|
|
"DESIRED: 3")
|
|
|
|
|
xfail_issue_48190 = xfail_test(reason="RobertaBase-11 - AssertionError: Items are not equal: "
|
|
|
|
|
"ACTUAL: dtype('float64') DESIRED: dtype('float32')")
|
2021-02-26 15:34:24 +01:00
|
|
|
xfail_issue_49750 = xfail_test(reason="RuntimeError: Unsupported dynamic ops: v4::Interpolate")
|
|
|
|
|
xfail_issue_49752 = xfail_test(reason="RuntimeError: Unsupported dynamic ops: v1::Pad")
|
|
|
|
|
xfail_issue_49753 = xfail_test(reason="RuntimeError: Unsupported dynamic ops: v1::StridedSlice")
|
|
|
|
|
xfail_issue_49754 = xfail_test(reason="RuntimeError: Unsupported dynamic ops: v1::TopKIE")
|
2021-04-12 12:06:29 +02:00
|
|
|
xfail_issue_52463 = xfail_test(reason="test_operator_add_size1_singleton_broadcast_cpu - "
|
|
|
|
|
"Not equal to tolerance")
|
2021-06-16 14:57:29 +02:00
|
|
|
xfail_issue_58033 = xfail_test(reason="Einsum operation misses support for complex ellipsis equations")
|
2021-06-28 16:05:54 +02:00
|
|
|
xfail_issue_58676 = xfail_test(reason="AssertionError: Not equal to tolerance rtol=0.001, atol=1e-07")
|
2021-06-07 13:01:20 +02:00
|
|
|
xfail_issue_onnx_models_140 = xfail_test(reason="https://github.com/onnx/models/issues/140")
|
2021-08-13 11:56:49 +03:00
|
|
|
xfail_issue_54630 = xfail_test(reason="Gather with negative indices is not yet implemented on CPU")
|
2021-08-25 11:37:44 +02:00
|
|
|
|
|
|
|
|
xfail_issue_63033 = xfail_test(reason="BatchNormalization: Training mode is not supported")
|
|
|
|
|
xfail_issue_63036 = xfail_test(reason="Changes in ConvTranspose padding")
|
|
|
|
|
xfail_issue_63039 = xfail_test(reason="Result mismatches with UINT8 operations")
|
|
|
|
|
xfail_issue_63043 = xfail_test(reason="Recurrent node expects constants as W, R, B inputs.")
|
|
|
|
|
xfail_issue_63044 = xfail_test(reason="ONNX opset 14 operation: Trilu")
|
|
|
|
|
|
|
|
|
|
skip_rng_tests = pytest.mark.skip(reason="Tests use random number generator with no seed.")
|
|
|
|
|
xfail_issue_63136 = xfail_test(reason="Unsupported operation: CastLike")
|
|
|
|
|
xfail_issue_63137 = xfail_test(reason="Unsupported operations: OptionalHasElement, OptionalGetElement")
|
|
|
|
|
xfail_issue_63138 = xfail_test(reason="Missing ONNX Shape-15 support")
|
2021-09-15 10:57:13 +02:00
|
|
|
xfail_issue_63643 = xfail_test(reason="RuntimeError: Unsupported operation of type: Convolution name")
|
2021-11-12 22:44:44 +03:00
|
|
|
xfail_issue_68212 = xfail_test(reason="Unsupported reading model with bytes streams")
|
