diff --git a/ngraph/frontend/onnx/frontend/src/core/transform.cpp b/ngraph/frontend/onnx/frontend/src/core/transform.cpp index 758216fed4a..c22157d8452 100644 --- a/ngraph/frontend/onnx/frontend/src/core/transform.cpp +++ b/ngraph/frontend/onnx/frontend/src/core/transform.cpp @@ -6,11 +6,35 @@ #include #include +#include #include "core/model.hpp" #include "ngraph/file_util.hpp" +#include "ngraph/log.hpp" #include "ops_bridge.hpp" +namespace ngraph { +namespace onnx_import { +namespace transform { +namespace detail { +ONNX_NAMESPACE::TypeProto get_input_type(std::string const& name, ONNX_NAMESPACE::GraphProto& graph) { + for (auto& input : graph.input()) { + if (input.name() == name) { + return input.type(); + } + } + for (auto& value_info : graph.value_info()) { + if (value_info.name() == name) { + return value_info.type(); + } + } + return ONNX_NAMESPACE::TypeProto(); +} +} // namespace detail +} // namespace transform +} // namespace onnx_import +} // namespace ngraph + void ngraph::onnx_import::transform::expand_onnx_functions(ONNX_NAMESPACE::ModelProto& model_proto) { auto graph_proto = model_proto.mutable_graph(); @@ -43,7 +67,19 @@ void ngraph::onnx_import::transform::expand_onnx_functions(ONNX_NAMESPACE::Model } else if (node_op_schema->HasContextDependentFunction()) { - ONNX_NAMESPACE::FunctionBodyBuildContextImpl ctx(node); + // In order to expand a context-dependent function, we need to infer types + try { + ONNX_NAMESPACE::shape_inference::InferShapes(model_proto); + } catch (const std::exception& e) { + NGRAPH_WARN << "ONNX Shape inference failed: " << e.what(); + } + + std::vector input_types; + for (const auto& input : node.input()) { + input_types.push_back(ngraph::onnx_import::transform::detail::get_input_type(input, *graph_proto)); + } + + ONNX_NAMESPACE::FunctionBodyBuildContextImpl ctx(node, input_types); ONNX_NAMESPACE::FunctionProto func_proto; node_op_schema->BuildContextDependentFunction(ctx, func_proto); ONNX_NAMESPACE::FunctionExpandHelper(node, func_proto, *graph_proto); diff --git a/ngraph/frontend/onnx/frontend/src/core/transform.hpp b/ngraph/frontend/onnx/frontend/src/core/transform.hpp index 2149f149625..722d95d80fa 100644 --- a/ngraph/frontend/onnx/frontend/src/core/transform.hpp +++ b/ngraph/frontend/onnx/frontend/src/core/transform.hpp @@ -9,6 +9,7 @@ namespace ngraph { namespace onnx_import { namespace transform { + /// \brief Replace external_data path in tensors with full path to data file. /// /// Paths to external data files are stored as relative to model path. @@ -19,7 +20,8 @@ namespace transform { /// \param model_path Filesystem path to the ONNX model file. void update_external_data_paths(ONNX_NAMESPACE::ModelProto& model_proto, const std::string& model_path); -static const std::vector onnx_functions_to_expand = {"Celu", +static const std::vector onnx_functions_to_expand = {"Bernoulli", + "Celu", "DynamicQuantizeLinear", "GreaterOrEqual", "LessOrEqual", @@ -63,7 +65,5 @@ static const std::vector legacy_ops_to_fixup = {"DeformableConv2D", void fixup_legacy_operators(ONNX_NAMESPACE::ModelProto& model_proto); } // namespace transform - } // namespace onnx_import - } // namespace ngraph diff --git a/ngraph/frontend/onnx/frontend/src/op/cast.cpp b/ngraph/frontend/onnx/frontend/src/op/cast.cpp index 82b252c63a6..6c083877497 100644 --- a/ngraph/frontend/onnx/frontend/src/op/cast.cpp +++ b/ngraph/frontend/onnx/frontend/src/op/cast.cpp @@ -14,6 +14,7 @@ namespace ngraph { namespace onnx_import { namespace op { namespace set_1 { + OutputVector cast(const Node& node) { auto data = node.get_ng_inputs().at(0); int64_t target_type = node.get_attribute_value("to"); @@ -23,9 +24,6 @@ OutputVector cast(const Node& node) { } } // namespace set_1 - } // namespace op - } // namespace onnx_import - } // namespace ngraph diff --git a/ngraph/frontend/onnx/frontend/src/op/cast.hpp b/ngraph/frontend/onnx/frontend/src/op/cast.hpp index eed57d51f9a..e712e7f5034 100644 --- a/ngraph/frontend/onnx/frontend/src/op/cast.hpp +++ b/ngraph/frontend/onnx/frontend/src/op/cast.hpp @@ -11,12 +11,10 @@ namespace ngraph { namespace onnx_import { namespace op { namespace set_1 { + OutputVector cast(const Node& node); } // namespace set_1 - } // namespace op - } // namespace onnx_import - } // namespace ngraph diff --git a/ngraph/frontend/onnx/frontend/src/op/cast_like.cpp b/ngraph/frontend/onnx/frontend/src/op/cast_like.cpp new file mode 100644 index 00000000000..112b55749e4 --- /dev/null +++ b/ngraph/frontend/onnx/frontend/src/op/cast_like.cpp @@ -0,0 +1,25 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include "default_opset.hpp" +#include "ngraph/type/element_type.hpp" +#include "op/cast.hpp" +#include "utils/common.hpp" + +namespace ngraph { +namespace onnx_import { +namespace op { +namespace set_1 { + +OutputVector cast_like(const Node& node) { + auto inputs = node.get_ng_inputs(); + return {std::make_shared(inputs.at(0), inputs.at(1))}; +} + +} // namespace set_1 +} // namespace op +} // namespace onnx_import +} // namespace ngraph diff --git a/ngraph/frontend/onnx/frontend/src/op/cast_like.hpp b/ngraph/frontend/onnx/frontend/src/op/cast_like.hpp new file mode 100644 index 00000000000..fb197b244ca --- /dev/null +++ b/ngraph/frontend/onnx/frontend/src/op/cast_like.hpp @@ -0,0 +1,20 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "ngraph/node.hpp" +#include "onnx_import/core/node.hpp" + +namespace ngraph { +namespace onnx_import { +namespace op { +namespace set_1 { + +OutputVector cast_like(const Node& node); + +} // namespace set_1 +} // namespace op +} // namespace onnx_import +} // namespace ngraph diff --git a/ngraph/frontend/onnx/frontend/src/op/hard_swish.hpp b/ngraph/frontend/onnx/frontend/src/op/hard_swish.hpp new file mode 100644 index 00000000000..68b2c403026 --- /dev/null +++ b/ngraph/frontend/onnx/frontend/src/op/hard_swish.hpp @@ -0,0 +1,24 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "default_opset.hpp" +#include "ngraph/node.hpp" +#include "onnx_import/core/node.hpp" + +namespace ngraph { +namespace onnx_import { +namespace op { +namespace set_1 { +inline OutputVector hard_swish(const Node& node) { + return {std::make_shared(node.get_ng_inputs().at(0))}; +} +} // namespace set_1 + +} // namespace op + +} // namespace onnx_import + +} // namespace ngraph diff --git a/ngraph/frontend/onnx/frontend/src/op/shape.cpp b/ngraph/frontend/onnx/frontend/src/op/shape.cpp index a35bbb235a1..04f99516c1d 100644 --- a/ngraph/frontend/onnx/frontend/src/op/shape.cpp +++ b/ngraph/frontend/onnx/frontend/src/op/shape.cpp @@ -15,15 +15,13 @@ namespace ngraph { namespace onnx_import { namespace op { namespace set_1 { + OutputVector shape(const Node& node) { const auto data = node.get_ng_inputs().at(0); return {std::make_shared(data)}; } } // namespace set_1 - } // namespace op - } // namespace onnx_import - } // namespace ngraph diff --git a/ngraph/frontend/onnx/frontend/src/op/shape.hpp b/ngraph/frontend/onnx/frontend/src/op/shape.hpp index c73465f9328..9124aa6f48f 100644 --- a/ngraph/frontend/onnx/frontend/src/op/shape.hpp +++ b/ngraph/frontend/onnx/frontend/src/op/shape.hpp @@ -11,12 +11,10 @@ namespace ngraph { namespace onnx_import { namespace op { namespace set_1 { + OutputVector shape(const Node& node); } // namespace set_1 - } // namespace op - } // namespace onnx_import - } // namespace ngraph diff --git a/ngraph/frontend/onnx/frontend/src/op/slice.cpp b/ngraph/frontend/onnx/frontend/src/op/slice.cpp index ce5d8240e63..13c08449cab 100644 --- a/ngraph/frontend/onnx/frontend/src/op/slice.cpp +++ b/ngraph/frontend/onnx/frontend/src/op/slice.cpp @@ -217,9 +217,6 @@ OutputVector slice(const Node& node) { } } // namespace set_1 - } // namespace op - } // namespace onnx_import - } // namespace ngraph diff --git a/ngraph/frontend/onnx/frontend/src/ops_bridge.cpp b/ngraph/frontend/onnx/frontend/src/ops_bridge.cpp index 6d5fd9b3467..ed759c1b5ea 100644 --- a/ngraph/frontend/onnx/frontend/src/ops_bridge.cpp +++ b/ngraph/frontend/onnx/frontend/src/ops_bridge.cpp @@ -26,6 +26,7 @@ #include "op/batch_norm.hpp" #include "op/bitshift.hpp" #include "op/cast.hpp" +#include "op/cast_like.hpp" #include "op/ceil.hpp" #include "op/clip.hpp" #include "op/compress.hpp" @@ -62,6 +63,7 @@ #include "op/greater.hpp" #include "op/gru.hpp" #include "op/hard_sigmoid.hpp" +#include "op/hard_swish.hpp" #include "op/hardmax.hpp" #include "op/identity.hpp" #include "op/image_scaler.hpp" @@ -285,6 +287,7 @@ OperatorsBridge::OperatorsBridge() { REGISTER_OPERATOR("BatchNormalization", 7, batch_norm); REGISTER_OPERATOR("BitShift", 1, bitshift); REGISTER_OPERATOR("Cast", 1, cast); + REGISTER_OPERATOR("CastLike", 1, cast_like); REGISTER_OPERATOR("Ceil", 1, ceil); REGISTER_OPERATOR("Clip", 1, clip); REGISTER_OPERATOR("Clip", 11, clip); @@ -330,6 +333,7 @@ OperatorsBridge::OperatorsBridge() { REGISTER_OPERATOR("Hardmax", 1, hardmax); REGISTER_OPERATOR("Hardmax", 13, hardmax); REGISTER_OPERATOR("HardSigmoid", 1, hard_sigmoid); + REGISTER_OPERATOR("HardSwish", 1, hard_swish); REGISTER_OPERATOR("Identity", 1, identity); REGISTER_OPERATOR("ImageScaler", 1, image_scaler); REGISTER_OPERATOR("InstanceNormalization", 1, instance_norm); diff --git a/ngraph/python/requirements_test.txt b/ngraph/python/requirements_test.txt index 0a63e05497a..595f3e4f75d 100644 --- a/ngraph/python/requirements_test.txt +++ b/ngraph/python/requirements_test.txt @@ -4,7 +4,7 @@ flake8-comprehensions==3.3.0 flake8-docstrings==1.6.0 flake8-quotes==3.2.0 mypy==0.812 -onnx==1.8.1 +onnx==1.10.1 pydocstyle==5.1.1 pytest-xdist==2.2.1 pytest==6.1.2 diff --git a/ngraph/python/tests/__init__.py b/ngraph/python/tests/__init__.py index a132f6fad53..929eb41658b 100644 --- a/ngraph/python/tests/__init__.py +++ b/ngraph/python/tests/__init__.py @@ -62,7 +62,7 @@ xfail_issue_38701 = xfail_test(reason="RuntimeError: unsupported element type: S xfail_issue_38706 = xfail_test(reason="RuntimeError: output_3.0 has zero dimension which is not allowed") xfail_issue_38708 = xfail_test(reason="RuntimeError: While validating ONNX node '': " "Axes input must be constant") -xfail_issue_38710 = xfail_test(reason="RuntimeError: roi has zero dimension which is not allowed") +xfail_issue_38710 = xfail_test(reason="RuntimeError: data has zero dimension which is not allowed") xfail_issue_38713 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" "ai.onnx.preview.training.Momentum") xfail_issue_43742 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:" @@ -91,16 +91,16 @@ xfail_issue_38735 = xfail_test(reason="RuntimeError: nGraph does not support the "ai.onnx.preview.training.Adagrad") xfail_issue_48052 = xfail_test(reason="Dropout op is not supported in traning mode") xfail_issue_45180 = xfail_test(reason="RuntimeError: Unsupported dynamic op: ReduceSum") -xfail_issue_44848 = xfail_test(reason="E Unsupported dynamic op: Range") -xfail_issue_44851 = xfail_test(reason="E Unsupported dynamic op: Broadcast") -xfail_issue_44854 = xfail_test(reason="E Unsupported dynamic op: VariadicSplit") -xfail_issue_44858 = xfail_test(reason="E Unsupported dynamic op: Unsqueeze") -xfail_issue_44956 = xfail_test(reason="E Unsupported dynamic op: Loop") -xfail_issue_44957 = xfail_test(reason="E Unsupported dynamic op: NonZero") -xfail_issue_44958 = xfail_test(reason="E Unsupported dynamic op: Interpolate") -xfail_issue_44965 = xfail_test(reason="E RuntimeError: value info has no element") -xfail_issue_44968 = xfail_test(reason="E Unsupported dynamic op: Squeeze") -xfail_issue_44976 = xfail_test(reason="E RuntimeError: Quantize layer with name:" +xfail_issue_44848 = xfail_test(reason="Expected: Unsupported dynamic op: Range") +xfail_issue_44851 = xfail_test(reason="Expected: Unsupported dynamic op: Broadcast") +xfail_issue_44854 = xfail_test(reason="Expected: Unsupported dynamic op: VariadicSplit") +xfail_issue_44858 = xfail_test(reason="Expected: Unsupported dynamic op: Unsqueeze") +xfail_issue_44956 = xfail_test(reason="Expected: Unsupported dynamic op: Loop") +xfail_issue_44957 = xfail_test(reason="Expected: Unsupported dynamic op: NonZero") +xfail_issue_44958 = xfail_test(reason="Expected: Unsupported dynamic op: Interpolate") +xfail_issue_44965 = xfail_test(reason="Expected: RuntimeError: value info has no element") +xfail_issue_44968 = xfail_test(reason="Expected: Unsupported dynamic op: Squeeze") +xfail_issue_44976 = xfail_test(reason="Expected: RuntimeError: Quantize layer with name:" "FakeQuantize_xxx has non const input on 1 port") xfail_issue_46762 = xfail_test(reason="Incorrect result of Minimum op if uint data type is used") xfail_issue_47323 = xfail_test(reason="RuntimeError: The plugin does not support FP64") @@ -145,5 +145,16 @@ xfail_issue_52463 = xfail_test(reason="test_operator_add_size1_singleton_broadca xfail_issue_58033 = xfail_test(reason="Einsum operation misses support for complex ellipsis equations") xfail_issue_58676 = xfail_test(reason="AssertionError: Not equal to tolerance rtol=0.001, atol=1e-07") xfail_issue_onnx_models_140 = xfail_test(reason="https://github.com/onnx/models/issues/140") - xfail_issue_54630 = xfail_test(reason="Gather with negative indices is not yet implemented on CPU") + +xfail_issue_63033 = xfail_test(reason="BatchNormalization: Training mode is not supported") +xfail_issue_63036 = xfail_test(reason="Changes in ConvTranspose padding") +xfail_issue_63039 = xfail_test(reason="Result mismatches with UINT8 operations") +xfail_issue_63043 = xfail_test(reason="Recurrent node expects constants as W, R, B inputs.") +xfail_issue_63044 = xfail_test(reason="ONNX opset 14 operation: Trilu") +xfail_issue_63045 = xfail_test(reason="Maxpool with strides, padding and dilations fail") + +skip_rng_tests = pytest.mark.skip(reason="Tests use random number generator with no seed.") +xfail_issue_63136 = xfail_test(reason="Unsupported operation: CastLike") +xfail_issue_63137 = xfail_test(reason="Unsupported operations: OptionalHasElement, OptionalGetElement") +xfail_issue_63138 = xfail_test(reason="Missing ONNX Shape-15 support") diff --git a/ngraph/python/tests/test_onnx/model_zoo_preprocess.sh b/ngraph/python/tests/test_onnx/model_zoo_preprocess.sh index dd35ed340eb..e7b5a891cf1 100755 --- a/ngraph/python/tests/test_onnx/model_zoo_preprocess.sh +++ b/ngraph/python/tests/test_onnx/model_zoo_preprocess.sh @@ -23,6 +23,8 @@ function print_help { echo " -s Onnx Model Zoo commit SHA" echo " -m update MSFT models" echo " -f force update of a chosen model" + echo "" + echo "Note: This script requires wget, GNU tar (not bsdtar) and git with LFS support." } while getopts "homfd:s:" opt; do @@ -70,7 +72,9 @@ function pull_and_postprocess_onnx_model_zoo() { find "$ONNX_MODELS_DIR" -name "*.onnx" | while read filename; do rm "$filename"; done; printf "Extracting tar.gz archives into %s\n" "$ONNX_MODELS_DIR" - find "$ONNX_MODELS_DIR" -name '*.tar.gz' -execdir sh -c 'BASEDIR=$(basename "{}" .tar.gz) && rm -rf $BASEDIR && mkdir -p $BASEDIR' \; -execdir sh -c 'BASEDIR=$(basename "{}" .tar.gz) && tar --warning=no-unknown-keyword -xvzf "{}" -C $BASEDIR' \; + find "$ONNX_MODELS_DIR" -name '*.tar.gz' \ + -execdir sh -c 'BASEDIR=$(basename "{}" .tar.gz) && rm -rf $BASEDIR && mkdir -p $BASEDIR' \; \ + -execdir sh -c 'BASEDIR=$(basename "{}" .tar.gz) && tar --warning=no-unknown-keyword -xvzf "{}" -C $BASEDIR' \; echo "Postprocessing of ONNX Model Zoo models:" @@ -129,7 +133,7 @@ function postprocess_msft_models() { } if [[ $ENABLE_ONNX_MODELS_ZOO = false ]] && [[ $ENABLE_MSFT_MODELS = false ]] ; then - printf "Please choose an option to update chosen model: + echo "Please choose an option to update chosen model: -o to update ONNX Model ZOO -m to update MSFT models" exit 170 @@ -144,7 +148,7 @@ fi # check if general model zoo directory exists (directory to store ONNX model zoo and MSFT models) if [[ ! -d $MODEL_ZOO_DIR ]] ; then - printf "The general model directory: %s doesn't exist on your filesystem then will be created \n" "$MODEL_ZOO_DIR" + printf "The general model directory: %s doesn't exist on your filesystem, it will be created \n" "$MODEL_ZOO_DIR" mkdir -p $MODEL_ZOO_DIR else printf "The general model directory: %s found\n" "$MODEL_ZOO_DIR" diff --git a/ngraph/python/tests/test_onnx/test_backend.py b/ngraph/python/tests/test_onnx/test_backend.py index f8a1d99c900..367e9a04dd5 100644 --- a/ngraph/python/tests/test_onnx/test_backend.py +++ b/ngraph/python/tests/test_onnx/test_backend.py @@ -4,64 +4,74 @@ import logging import onnx.backend.test - +from tests import ( + BACKEND_NAME, + skip_rng_tests, + xfail_issue_33488, + xfail_issue_33535, + xfail_issue_33538, + xfail_issue_33581, + xfail_issue_33589, + xfail_issue_33593, + xfail_issue_33595, + xfail_issue_33596, + xfail_issue_33606, + xfail_issue_33633, + xfail_issue_33651, + xfail_issue_38091, + xfail_issue_38699, + xfail_issue_38701, + xfail_issue_38706, + xfail_issue_38708, + xfail_issue_38710, + xfail_issue_38713, + xfail_issue_38722, + xfail_issue_38723, + xfail_issue_38724, + xfail_issue_38732, + xfail_issue_38734, + xfail_issue_38735, + xfail_issue_39656, + xfail_issue_39658, + xfail_issue_39659, + xfail_issue_39662, + xfail_issue_43742, + xfail_issue_44848, + xfail_issue_44851, + xfail_issue_44854, + xfail_issue_44858, + xfail_issue_44956, + xfail_issue_44957, + xfail_issue_44958, + xfail_issue_44965, + xfail_issue_44968, + xfail_issue_44976, + xfail_issue_45180, + xfail_issue_45344, + xfail_issue_46762, + xfail_issue_47323, + xfail_issue_47337, + xfail_issue_48052, + xfail_issue_49207, + xfail_issue_49750, + xfail_issue_49752, + xfail_issue_49753, + xfail_issue_49754, + xfail_issue_52463, + xfail_issue_55760, + xfail_issue_58033, + xfail_issue_63033, + xfail_issue_63036, + xfail_issue_63039, + xfail_issue_63043, + xfail_issue_63044, + xfail_issue_63045, + xfail_issue_63136, + xfail_issue_63137, + xfail_issue_63138, +) from tests.test_onnx.utils.onnx_backend import OpenVinoTestBackend -from tests import (BACKEND_NAME, - xfail_issue_33488, - xfail_issue_33535, - xfail_issue_33538, - xfail_issue_33581, - xfail_issue_33589, - xfail_issue_33593, - xfail_issue_33595, - xfail_issue_33596, - xfail_issue_33606, - xfail_issue_33633, - xfail_issue_33651, - xfail_issue_38091, - xfail_issue_38699, - xfail_issue_38701, - xfail_issue_38706, - xfail_issue_38708, - xfail_issue_38713, - xfail_issue_38722, - xfail_issue_38723, - xfail_issue_38724, - xfail_issue_38732, - xfail_issue_38734, - xfail_issue_38735, - xfail_issue_39656, - xfail_issue_39658, - xfail_issue_39659, - xfail_issue_39662, - xfail_issue_43742, - xfail_issue_44848, - xfail_issue_44851, - xfail_issue_44854, - xfail_issue_44858, - xfail_issue_44956, - xfail_issue_44957, - xfail_issue_44958, - xfail_issue_44965, - xfail_issue_44968, - xfail_issue_44976, - xfail_issue_45180, - xfail_issue_45344, - xfail_issue_46762, - xfail_issue_47323, - xfail_issue_47337, - xfail_issue_48052, - xfail_issue_49207, - xfail_issue_49750, - xfail_issue_49752, - xfail_issue_49753, - xfail_issue_49754, - xfail_issue_52463, - xfail_issue_55760, - xfail_issue_58033, - ) - def expect_fail(test_case_path, xfail): # type: (str) -> None """Mark the test as expected to fail.""" @@ -70,7 +80,9 @@ def expect_fail(test_case_path, xfail): # type: (str) -> None if hasattr(module, test_name): xfail(getattr(module, test_name)) else: - logging.getLogger().warning("Could not mark test as XFAIL, not found: %s", test_case_path) + logging.getLogger().warning( + "Could not mark test as XFAIL, not found: %s", test_case_path + ) OpenVinoTestBackend.backend_name = BACKEND_NAME @@ -107,265 +119,435 @@ OnnxBackendPyTorchConvertedModelTest = None globals().update(backend_test.enable_report().test_cases) tests_expected_to_fail = [ - (xfail_issue_49207, - "OnnxBackendNodeModelTest.test_rnn_seq_length_cpu", - "OnnxBackendNodeModelTest.test_simple_rnn_defaults_cpu", - "OnnxBackendNodeModelTest.test_simple_rnn_with_initial_bias_cpu", - "OnnxBackendNodeModelTest.test_gru_defaults_cpu", - "OnnxBackendNodeModelTest.test_gru_seq_length_cpu", - "OnnxBackendNodeModelTest.test_gru_with_initial_bias_cpu", - "OnnxBackendNodeModelTest.test_lstm_defaults_cpu", - "OnnxBackendNodeModelTest.test_lstm_with_initial_bias_cpu", - "OnnxBackendNodeModelTest.test_lstm_with_peepholes_cpu"), - (xfail_issue_49752, - "OnnxBackendNodeModelTest.test_constant_pad_cpu", - "OnnxBackendNodeModelTest.test_edge_pad_cpu", - "OnnxBackendNodeModelTest.test_reflect_pad_cpu"), - (xfail_issue_39656, - "OnnxBackendNodeModelTest.test_reshape_extended_dims_cpu", - "OnnxBackendNodeModelTest.test_reshape_negative_dim_cpu", - "OnnxBackendNodeModelTest.test_reshape_one_dim_cpu", - "OnnxBackendNodeModelTest.test_reshape_reduced_dims_cpu", - "OnnxBackendNodeModelTest.test_reshape_negative_extended_dims_cpu", - "OnnxBackendNodeModelTest.test_reshape_reordered_all_dims_cpu", - "OnnxBackendNodeModelTest.test_reshape_reordered_last_dims_cpu", - "OnnxBackendNodeModelTest.test_reshape_zero_and_negative_dim_cpu", - "OnnxBackendNodeModelTest.test_reshape_zero_dim_cpu"), - (xfail_issue_39658, - "OnnxBackendNodeModelTest.test_tile_cpu", - "OnnxBackendNodeModelTest.test_tile_precomputed_cpu"), - (xfail_issue_39659, - "OnnxBackendNodeModelTest.test_constantofshape_float_ones_cpu", - "OnnxBackendNodeModelTest.test_constantofshape_int_zeros_cpu", - "OnnxBackendNodeModelTest.test_constantofshape_int_shape_zero_cpu"), - (xfail_issue_45344, - "OnnxBackendNodeModelTest.test_nonmaxsuppression_center_point_box_format_cpu", - "OnnxBackendNodeModelTest.test_nonmaxsuppression_flipped_coordinates_cpu", - "OnnxBackendNodeModelTest.test_nonmaxsuppression_identical_boxes_cpu", - "OnnxBackendNodeModelTest.test_nonmaxsuppression_limit_output_size_cpu", - "OnnxBackendNodeModelTest.test_nonmaxsuppression_single_box_cpu", - "OnnxBackendNodeModelTest.test_nonmaxsuppression_suppress_by_IOU_and_scores_cpu", - "OnnxBackendNodeModelTest.test_nonmaxsuppression_suppress_by_IOU_cpu", - "OnnxBackendNodeModelTest.test_nonmaxsuppression_two_batches_cpu", - "OnnxBackendNodeModelTest.test_nonmaxsuppression_two_classes_cpu"), - (xfail_issue_39662, - "OnnxBackendNodeModelTest.test_nonmaxsuppression_two_classes_cpu", - "OnnxBackendNodeModelTest.test_scatter_elements_with_negative_indices_cpu", - "OnnxBackendNodeModelTest.test_gather_negative_indices_cpu"), - (xfail_issue_49753, - "OnnxBackendNodeModelTest.test_slice_default_axes_cpu"), - (xfail_issue_49754, - "OnnxBackendNodeModelTest.test_top_k_cpu", - "OnnxBackendNodeModelTest.test_top_k_negative_axis_cpu", - "OnnxBackendNodeModelTest.test_top_k_smallest_cpu"), - (xfail_issue_33633, - "OnnxBackendNodeModelTest.test_maxpool_2d_dilations_cpu"), - (xfail_issue_46762, - "OnnxBackendNodeModelTest.test_min_uint8_cpu", - "OnnxBackendNodeModelTest.test_min_uint16_cpu", - "OnnxBackendNodeModelTest.test_min_uint32_cpu", - "OnnxBackendNodeModelTest.test_min_uint64_cpu"), - (xfail_issue_55760, - "OnnxBackendNodeModelTest.test_argmax_negative_axis_keepdims_example_select_last_index_cpu", - "OnnxBackendNodeModelTest.test_argmin_negative_axis_keepdims_example_select_last_index_cpu", - "OnnxBackendNodeModelTest.test_argmax_negative_axis_keepdims_random_select_last_index_cpu", - "OnnxBackendNodeModelTest.test_argmin_negative_axis_keepdims_random_select_last_index_cpu"), - (xfail_issue_38091, - "OnnxBackendNodeModelTest.test_gather_negative_indices_cpu"), - (xfail_issue_52463, - "OnnxBackendPyTorchOperatorModelTest.test_operator_add_size1_singleton_broadcast_cpu"), - (xfail_issue_47323, - "OnnxBackendPyTorchOperatorModelTest.test_operator_add_broadcast_cpu", - "OnnxBackendPyTorchOperatorModelTest.test_operator_addconstant_cpu", - "OnnxBackendPyTorchOperatorModelTest.test_operator_add_size1_right_broadcast_cpu"), - (xfail_issue_38699, - "OnnxBackendSimpleModelTest.test_gradient_of_add_and_mul_cpu", - "OnnxBackendSimpleModelTest.test_gradient_of_add_cpu"), - (xfail_issue_33596, - "OnnxBackendSimpleModelTest.test_sequence_model5_cpu", - "OnnxBackendSimpleModelTest.test_sequence_model7_cpu", - "OnnxBackendSimpleModelTest.test_sequence_model1_cpu", - "OnnxBackendSimpleModelTest.test_sequence_model3_cpu", - "OnnxBackendSimpleModelTest.test_sequence_model6_cpu", - "OnnxBackendSimpleModelTest.test_sequence_model8_cpu", - "OnnxBackendSimpleModelTest.test_sequence_model4_cpu", - "OnnxBackendSimpleModelTest.test_sequence_model2_cpu"), - (xfail_issue_38701, - "OnnxBackendSimpleModelTest.test_strnorm_model_monday_casesensintive_nochangecase_cpu", - "OnnxBackendSimpleModelTest.test_strnorm_model_nostopwords_nochangecase_cpu", - "OnnxBackendSimpleModelTest.test_strnorm_model_monday_empty_output_cpu", - "OnnxBackendSimpleModelTest.test_strnorm_model_monday_insensintive_upper_twodim_cpu", - "OnnxBackendSimpleModelTest.test_strnorm_model_monday_casesensintive_lower_cpu", - "OnnxBackendSimpleModelTest.test_strnorm_model_monday_casesensintive_upper_cpu", - "OnnxBackendNodeModelTest.test_strnormalizer_nostopwords_nochangecase_cpu", - "OnnxBackendNodeModelTest.test_strnormalizer_export_monday_casesensintive_nochangecase_cpu", - "OnnxBackendNodeModelTest.test_strnormalizer_export_monday_insensintive_upper_twodim_cpu", - "OnnxBackendNodeModelTest.test_strnormalizer_export_monday_casesensintive_lower_cpu", - "OnnxBackendNodeModelTest.test_strnormalizer_export_monday_empty_output_cpu", - "OnnxBackendNodeModelTest.test_strnormalizer_export_monday_casesensintive_upper_cpu", - "OnnxBackendNodeModelTest.test_cast_STRING_to_FLOAT_cpu", - "OnnxBackendNodeModelTest.test_cast_FLOAT_to_STRING_cpu"), - (xfail_issue_33595, - "OnnxBackendNodeModelTest.test_unique_not_sorted_without_axis_cpu", - "OnnxBackendNodeModelTest.test_unique_sorted_with_negative_axis_cpu", - "OnnxBackendNodeModelTest.test_unique_sorted_with_axis_cpu", - "OnnxBackendNodeModelTest.test_unique_sorted_with_axis_3d_cpu", - "OnnxBackendNodeModelTest.test_unique_sorted_without_axis_cpu"), - (xfail_issue_33651, - "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_batch_onlybigrams_skip5_cpu", - "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_onlybigrams_levelempty_cpu", - "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_batch_onlybigrams_skip0_cpu", - "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_uniandbigrams_skip5_cpu", - "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_only_bigrams_skip0_cpu", - "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_batch_uniandbigrams_skip5_cpu", - "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_onlybigrams_skip5_cpu"), - (xfail_issue_38706, - "OnnxBackendNodeModelTest.test_split_zero_size_splits_cpu"), - (xfail_issue_38708, - "OnnxBackendNodeModelTest.test_slice_default_steps_cpu", - "OnnxBackendNodeModelTest.test_slice_negative_axes_cpu", - "OnnxBackendNodeModelTest.test_slice_neg_steps_cpu", - "OnnxBackendNodeModelTest.test_slice_neg_cpu", - "OnnxBackendNodeModelTest.test_slice_cpu", - "OnnxBackendNodeModelTest.test_slice_end_out_of_bounds_cpu", - "OnnxBackendNodeModelTest.test_slice_start_out_of_bounds_cpu"), - (xfail_issue_33538, - "OnnxBackendNodeModelTest.test_scan_sum_cpu", - "OnnxBackendNodeModelTest.test_scan9_sum_cpu"), - (xfail_issue_49750, - "OnnxBackendNodeModelTest.test_resize_downsample_scales_linear_align_corners_cpu", - "OnnxBackendNodeModelTest.test_resize_upsample_scales_cubic_cpu", - "OnnxBackendNodeModelTest.test_resize_downsample_scales_cubic_cpu", - "OnnxBackendNodeModelTest.test_resize_downsample_scales_cubic_align_corners_cpu", - "OnnxBackendNodeModelTest.test_resize_upsample_scales_nearest_cpu", - "OnnxBackendNodeModelTest.test_resize_upsample_scales_cubic_asymmetric_cpu", - "OnnxBackendNodeModelTest.test_resize_downsample_scales_cubic_A_n0p5_exclude_outside_cpu", - "OnnxBackendNodeModelTest.test_resize_upsample_scales_linear_cpu", - "OnnxBackendNodeModelTest.test_resize_upsample_scales_cubic_align_corners_cpu", - "OnnxBackendNodeModelTest.test_resize_upsample_scales_linear_align_corners_cpu", - "OnnxBackendNodeModelTest.test_resize_upsample_scales_cubic_A_n0p5_exclude_outside_cpu", - "OnnxBackendNodeModelTest.test_resize_downsample_scales_nearest_cpu", - "OnnxBackendNodeModelTest.test_resize_downsample_scales_linear_cpu", - "OnnxBackendNodeModelTest.test_resize_downsample_sizes_cubic_cpu", - "OnnxBackendNodeModelTest.test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu", - "OnnxBackendNodeModelTest.test_resize_upsample_sizes_nearest_floor_align_corners_cpu", - "OnnxBackendNodeModelTest.test_resize_upsample_sizes_nearest_cpu", - "OnnxBackendNodeModelTest.test_resize_upsample_sizes_nearest_ceil_half_pixel_cpu", - "OnnxBackendNodeModelTest.test_resize_upsample_sizes_cubic_cpu", - "OnnxBackendNodeModelTest.test_resize_downsample_sizes_linear_pytorch_half_pixel_cpu", - "OnnxBackendNodeModelTest.test_resize_downsample_sizes_nearest_cpu", - "OnnxBackendNodeModelTest.test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn_cpu"), - (xfail_issue_33581, - "OnnxBackendNodeModelTest.test_gather_elements_negative_indices_cpu"), - (xfail_issue_38713, - "OnnxBackendNodeModelTest.test_momentum_cpu", - "OnnxBackendNodeModelTest.test_nesterov_momentum_cpu", - "OnnxBackendNodeModelTest.test_momentum_multiple_cpu"), - (xfail_issue_47337, - "OnnxBackendNodeModelTest.test_onehot_without_axis_cpu", - "OnnxBackendNodeModelTest.test_onehot_with_negative_axis_cpu", - "OnnxBackendNodeModelTest.test_onehot_with_axis_cpu", - "OnnxBackendNodeModelTest.test_onehot_negative_indices_cpu"), - (xfail_issue_33488, - "OnnxBackendNodeModelTest.test_maxunpool_export_with_output_shape_cpu", - "OnnxBackendNodeModelTest.test_maxunpool_export_without_output_shape_cpu"), - (xfail_issue_33589, - "OnnxBackendNodeModelTest.test_isnan_cpu", - "OnnxBackendNodeModelTest.test_isinf_positive_cpu", - "OnnxBackendNodeModelTest.test_isinf_negative_cpu", - "OnnxBackendNodeModelTest.test_isinf_cpu"), - (xfail_issue_33535, - "OnnxBackendNodeModelTest.test_dynamicquantizelinear_min_adjusted_cpu", - "OnnxBackendNodeModelTest.test_dynamicquantizelinear_cpu", - "OnnxBackendNodeModelTest.test_dynamicquantizelinear_max_adjusted_cpu"), - (xfail_issue_38722, - "OnnxBackendNodeModelTest.test_matmulinteger_cpu", - "OnnxBackendNodeModelTest.test_qlinearmatmul_2D_cpu", - "OnnxBackendNodeModelTest.test_qlinearmatmul_3D_cpu"), - (xfail_issue_38723, - "OnnxBackendNodeModelTest.test_qlinearconv_cpu"), - (xfail_issue_38724, - "OnnxBackendNodeModelTest.test_resize_tf_crop_and_resize_cpu"), - (xfail_issue_33606, - "OnnxBackendNodeModelTest.test_det_2d_cpu", - "OnnxBackendNodeModelTest.test_det_nd_cpu"), - (xfail_issue_38732, - "OnnxBackendNodeModelTest.test_convinteger_with_padding_cpu", - "OnnxBackendNodeModelTest.test_basic_convinteger_cpu"), - (xfail_issue_38734, - "OnnxBackendNodeModelTest.test_adam_multiple_cpu", - "OnnxBackendNodeModelTest.test_adam_cpu"), - (xfail_issue_38735, - "OnnxBackendNodeModelTest.test_adagrad_multiple_cpu", - "OnnxBackendNodeModelTest.test_adagrad_cpu"), - (xfail_issue_48052, - "OnnxBackendNodeModelTest.test_training_dropout_cpu", - "OnnxBackendNodeModelTest.test_training_dropout_mask_cpu", - "OnnxBackendNodeModelTest.test_training_dropout_default_cpu", - "OnnxBackendNodeModelTest.test_training_dropout_zero_ratio_cpu", - "OnnxBackendNodeModelTest.test_training_dropout_default_mask_cpu", - "OnnxBackendNodeModelTest.test_training_dropout_zero_ratio_mask_cpu"), - (xfail_issue_45180, - "OnnxBackendNodeModelTest.test_reduce_sum_do_not_keepdims_example_cpu", - "OnnxBackendNodeModelTest.test_reduce_sum_do_not_keepdims_random_cpu", - "OnnxBackendNodeModelTest.test_reduce_sum_keepdims_example_cpu", - "OnnxBackendNodeModelTest.test_reduce_sum_keepdims_random_cpu", - "OnnxBackendNodeModelTest.test_reduce_sum_negative_axes_keepdims_example_cpu"), - (xfail_issue_43742, - "OnnxBackendNodeModelTest.test_if_cpu", - "OnnxBackendNodeModelTest.test_if_seq_cpu"), - (xfail_issue_44848, - "OnnxBackendNodeModelTest.test_range_float_type_positive_delta_cpu", - "OnnxBackendNodeModelTest.test_range_int32_type_negative_delta_cpu",), - (xfail_issue_44851, - "OnnxBackendNodeModelTest.test_expand_dim_changed_cpu", - "OnnxBackendNodeModelTest.test_expand_dim_unchanged_cpu", - "OnnxBackendSimpleModelTest.test_expand_shape_model1_cpu", - "OnnxBackendSimpleModelTest.test_expand_shape_model2_cpu", - "OnnxBackendSimpleModelTest.test_expand_shape_model3_cpu", - "OnnxBackendSimpleModelTest.test_expand_shape_model4_cpu",), - (xfail_issue_44854, - "OnnxBackendNodeModelTest.test_split_variable_parts_1d_cpu", - "OnnxBackendNodeModelTest.test_split_variable_parts_2d_cpu", - "OnnxBackendNodeModelTest.test_split_variable_parts_default_axis_cpu",), - (xfail_issue_44858, - "OnnxBackendNodeModelTest.test_unsqueeze_axis_0_cpu", - "OnnxBackendNodeModelTest.test_unsqueeze_axis_1_cpu", - "OnnxBackendNodeModelTest.test_unsqueeze_axis_2_cpu", - "OnnxBackendNodeModelTest.test_unsqueeze_negative_axes_cpu", - "OnnxBackendNodeModelTest.test_unsqueeze_three_axes_cpu", - "OnnxBackendNodeModelTest.test_unsqueeze_two_axes_cpu", - "OnnxBackendNodeModelTest.test_unsqueeze_unsorted_axes_cpu",), - (xfail_issue_44956, - "OnnxBackendNodeModelTest.test_loop11_cpu", - "OnnxBackendNodeModelTest.test_range_int32_type_negative_delta_expanded_cpu", - "OnnxBackendNodeModelTest.test_range_float_type_positive_delta_expanded_cpu"), - (xfail_issue_44957, - "OnnxBackendNodeModelTest.test_compress_0_cpu", - "OnnxBackendNodeModelTest.test_compress_1_cpu", - "OnnxBackendNodeModelTest.test_compress_default_axis_cpu", - "OnnxBackendNodeModelTest.test_compress_negative_axis_cpu", - "OnnxBackendNodeModelTest.test_nonzero_example_cpu"), - (xfail_issue_44958, - "OnnxBackendNodeModelTest.test_upsample_nearest_cpu"), - (xfail_issue_44965, - "OnnxBackendNodeModelTest.test_loop13_seq_cpu", - "OnnxBackendNodeModelTest.test_sequence_insert_at_back_cpu", - "OnnxBackendNodeModelTest.test_sequence_insert_at_front_cpu",), - (xfail_issue_44968, - "OnnxBackendNodeModelTest.test_squeeze_cpu", - "OnnxBackendNodeModelTest.test_squeeze_negative_axes_cpu",), - (xfail_issue_44976, - "OnnxBackendNodeModelTest.test_quantizelinear_axis_cpu", - "OnnxBackendNodeModelTest.test_dynamicquantizelinear_min_adjusted_expanded_cpu", - "OnnxBackendNodeModelTest.test_dynamicquantizelinear_expanded_cpu", - "OnnxBackendNodeModelTest.test_dynamicquantizelinear_max_adjusted_expanded_cpu", - "OnnxBackendNodeModelTest.test_quantizelinear_cpu"), - (xfail_issue_33593, - "OnnxBackendNodeModelTest.test_maxpool_with_argmax_2d_precomputed_strides_cpu", - "OnnxBackendNodeModelTest.test_maxpool_with_argmax_2d_precomputed_pads_cpu",), - (xfail_issue_58033, - "OnnxBackendNodeModelTest.test_einsum_batch_diagonal_cpu"), + ( + xfail_issue_49207, + "OnnxBackendNodeModelTest.test_rnn_seq_length_cpu", + "OnnxBackendNodeModelTest.test_simple_rnn_defaults_cpu", + "OnnxBackendNodeModelTest.test_simple_rnn_with_initial_bias_cpu", + "OnnxBackendNodeModelTest.test_gru_defaults_cpu", + "OnnxBackendNodeModelTest.test_gru_seq_length_cpu", + "OnnxBackendNodeModelTest.test_gru_with_initial_bias_cpu", + "OnnxBackendNodeModelTest.test_lstm_defaults_cpu", + "OnnxBackendNodeModelTest.test_lstm_with_initial_bias_cpu", + "OnnxBackendNodeModelTest.test_lstm_with_peepholes_cpu", + ), + ( + xfail_issue_49752, + "OnnxBackendNodeModelTest.test_constant_pad_cpu", + "OnnxBackendNodeModelTest.test_edge_pad_cpu", + "OnnxBackendNodeModelTest.test_reflect_pad_cpu", + ), + ( + xfail_issue_39656, + "OnnxBackendNodeModelTest.test_reshape_extended_dims_cpu", + "OnnxBackendNodeModelTest.test_reshape_negative_dim_cpu", + "OnnxBackendNodeModelTest.test_reshape_one_dim_cpu", + "OnnxBackendNodeModelTest.test_reshape_reduced_dims_cpu", + "OnnxBackendNodeModelTest.test_reshape_negative_extended_dims_cpu", + "OnnxBackendNodeModelTest.test_reshape_reordered_all_dims_cpu", + "OnnxBackendNodeModelTest.test_reshape_reordered_last_dims_cpu", + "OnnxBackendNodeModelTest.test_reshape_zero_and_negative_dim_cpu", + "OnnxBackendNodeModelTest.test_reshape_zero_dim_cpu", + ), + ( + xfail_issue_39658, + "OnnxBackendNodeModelTest.test_tile_cpu", + "OnnxBackendNodeModelTest.test_tile_precomputed_cpu", + ), + ( + xfail_issue_39659, + "OnnxBackendNodeModelTest.test_constantofshape_float_ones_cpu", + "OnnxBackendNodeModelTest.test_constantofshape_int_zeros_cpu", + "OnnxBackendNodeModelTest.test_constantofshape_int_shape_zero_cpu", + ), + ( + xfail_issue_45344, + "OnnxBackendNodeModelTest.test_nonmaxsuppression_center_point_box_format_cpu", + "OnnxBackendNodeModelTest.test_nonmaxsuppression_flipped_coordinates_cpu", + "OnnxBackendNodeModelTest.test_nonmaxsuppression_identical_boxes_cpu", + "OnnxBackendNodeModelTest.test_nonmaxsuppression_limit_output_size_cpu", + "OnnxBackendNodeModelTest.test_nonmaxsuppression_single_box_cpu", + "OnnxBackendNodeModelTest.test_nonmaxsuppression_suppress_by_IOU_and_scores_cpu", + "OnnxBackendNodeModelTest.test_nonmaxsuppression_suppress_by_IOU_cpu", + "OnnxBackendNodeModelTest.test_nonmaxsuppression_two_batches_cpu", + "OnnxBackendNodeModelTest.test_nonmaxsuppression_two_classes_cpu", + ), + ( + xfail_issue_39662, + "OnnxBackendNodeModelTest.test_nonmaxsuppression_two_classes_cpu", + "OnnxBackendNodeModelTest.test_scatter_elements_with_negative_indices_cpu", + "OnnxBackendNodeModelTest.test_gather_negative_indices_cpu", + ), + (xfail_issue_49753, "OnnxBackendNodeModelTest.test_slice_default_axes_cpu"), + ( + xfail_issue_49754, + "OnnxBackendNodeModelTest.test_top_k_cpu", + "OnnxBackendNodeModelTest.test_top_k_negative_axis_cpu", + "OnnxBackendNodeModelTest.test_top_k_smallest_cpu", + ), + (xfail_issue_33633, "OnnxBackendNodeModelTest.test_maxpool_2d_dilations_cpu"), + ( + xfail_issue_46762, + "OnnxBackendNodeModelTest.test_min_uint8_cpu", + "OnnxBackendNodeModelTest.test_min_uint16_cpu", + "OnnxBackendNodeModelTest.test_min_uint32_cpu", + "OnnxBackendNodeModelTest.test_min_uint64_cpu", + ), + ( + xfail_issue_55760, + "OnnxBackendNodeModelTest.test_argmax_negative_axis_keepdims_example_select_last_index_cpu", + "OnnxBackendNodeModelTest.test_argmin_negative_axis_keepdims_example_select_last_index_cpu", + "OnnxBackendNodeModelTest.test_argmax_negative_axis_keepdims_random_select_last_index_cpu", + "OnnxBackendNodeModelTest.test_argmin_negative_axis_keepdims_random_select_last_index_cpu", + ), + (xfail_issue_38091, "OnnxBackendNodeModelTest.test_gather_negative_indices_cpu"), + ( + xfail_issue_52463, + "OnnxBackendPyTorchOperatorModelTest.test_operator_add_size1_singleton_broadcast_cpu", + ), + ( + xfail_issue_47323, + "OnnxBackendPyTorchOperatorModelTest.test_operator_add_broadcast_cpu", + "OnnxBackendPyTorchOperatorModelTest.test_operator_addconstant_cpu", + "OnnxBackendPyTorchOperatorModelTest.test_operator_add_size1_right_broadcast_cpu", + ), + ( + xfail_issue_38699, + "OnnxBackendSimpleModelTest.test_gradient_of_add_and_mul_cpu", + "OnnxBackendSimpleModelTest.test_gradient_of_add_cpu", + ), + ( + xfail_issue_33596, + "OnnxBackendSimpleModelTest.test_sequence_model5_cpu", + "OnnxBackendSimpleModelTest.test_sequence_model7_cpu", + "OnnxBackendSimpleModelTest.test_sequence_model1_cpu", + "OnnxBackendSimpleModelTest.test_sequence_model3_cpu", + "OnnxBackendSimpleModelTest.test_sequence_model6_cpu", + "OnnxBackendSimpleModelTest.test_sequence_model8_cpu", + "OnnxBackendSimpleModelTest.test_sequence_model4_cpu", + "OnnxBackendSimpleModelTest.test_sequence_model2_cpu", + "OnnxBackendNodeModelTest.test_identity_sequence_cpu", + ), + ( + xfail_issue_38701, + "OnnxBackendSimpleModelTest.test_strnorm_model_monday_casesensintive_nochangecase_cpu", + "OnnxBackendSimpleModelTest.test_strnorm_model_nostopwords_nochangecase_cpu", + "OnnxBackendSimpleModelTest.test_strnorm_model_monday_empty_output_cpu", + "OnnxBackendSimpleModelTest.test_strnorm_model_monday_insensintive_upper_twodim_cpu", + "OnnxBackendSimpleModelTest.test_strnorm_model_monday_casesensintive_lower_cpu", + "OnnxBackendSimpleModelTest.test_strnorm_model_monday_casesensintive_upper_cpu", + "OnnxBackendNodeModelTest.test_strnormalizer_nostopwords_nochangecase_cpu", + "OnnxBackendNodeModelTest.test_strnormalizer_export_monday_casesensintive_nochangecase_cpu", + "OnnxBackendNodeModelTest.test_strnormalizer_export_monday_insensintive_upper_twodim_cpu", + "OnnxBackendNodeModelTest.test_strnormalizer_export_monday_casesensintive_lower_cpu", + "OnnxBackendNodeModelTest.test_strnormalizer_export_monday_empty_output_cpu", + "OnnxBackendNodeModelTest.test_strnormalizer_export_monday_casesensintive_upper_cpu", + "OnnxBackendNodeModelTest.test_cast_STRING_to_FLOAT_cpu", + "OnnxBackendNodeModelTest.test_cast_FLOAT_to_STRING_cpu", + "OnnxBackendNodeModelTest.test_castlike_FLOAT_to_STRING_cpu", + "OnnxBackendNodeModelTest.test_castlike_FLOAT_to_STRING_expanded_cpu", + "OnnxBackendNodeModelTest.test_castlike_STRING_to_FLOAT_cpu", + "OnnxBackendNodeModelTest.test_castlike_STRING_to_FLOAT_expanded_cpu", + ), + ( + xfail_issue_33595, + "OnnxBackendNodeModelTest.test_unique_not_sorted_without_axis_cpu", + "OnnxBackendNodeModelTest.test_unique_sorted_with_negative_axis_cpu", + "OnnxBackendNodeModelTest.test_unique_sorted_with_axis_cpu", + "OnnxBackendNodeModelTest.test_unique_sorted_with_axis_3d_cpu", + "OnnxBackendNodeModelTest.test_unique_sorted_without_axis_cpu", + ), + ( + xfail_issue_33651, + "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_batch_onlybigrams_skip5_cpu", + "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_onlybigrams_levelempty_cpu", + "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_batch_onlybigrams_skip0_cpu", + "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_uniandbigrams_skip5_cpu", + "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_only_bigrams_skip0_cpu", + "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_batch_uniandbigrams_skip5_cpu", + "OnnxBackendNodeModelTest.test_tfidfvectorizer_tf_onlybigrams_skip5_cpu", + ), + (xfail_issue_38706, "OnnxBackendNodeModelTest.test_split_zero_size_splits_cpu"), + ( + xfail_issue_38708, + "OnnxBackendNodeModelTest.test_slice_default_steps_cpu", + "OnnxBackendNodeModelTest.test_slice_negative_axes_cpu", + "OnnxBackendNodeModelTest.test_slice_neg_steps_cpu", + "OnnxBackendNodeModelTest.test_slice_neg_cpu", + "OnnxBackendNodeModelTest.test_slice_cpu", + "OnnxBackendNodeModelTest.test_slice_end_out_of_bounds_cpu", + "OnnxBackendNodeModelTest.test_slice_start_out_of_bounds_cpu", + ), + ( + xfail_issue_33538, + "OnnxBackendNodeModelTest.test_scan_sum_cpu", + "OnnxBackendNodeModelTest.test_scan9_sum_cpu", + ), + ( + xfail_issue_49750, + "OnnxBackendNodeModelTest.test_resize_downsample_scales_linear_align_corners_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_scales_cubic_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_scales_cubic_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_scales_cubic_align_corners_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_scales_nearest_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_scales_cubic_asymmetric_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_scales_cubic_A_n0p5_exclude_outside_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_scales_linear_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_scales_cubic_align_corners_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_scales_linear_align_corners_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_scales_cubic_A_n0p5_exclude_outside_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_scales_nearest_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_scales_linear_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_sizes_cubic_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_sizes_nearest_floor_align_corners_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_sizes_nearest_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_sizes_nearest_ceil_half_pixel_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_sizes_cubic_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_sizes_linear_pytorch_half_pixel_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_sizes_nearest_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn_cpu", + ), + ( + xfail_issue_33581, + "OnnxBackendNodeModelTest.test_gather_elements_negative_indices_cpu", + ), + ( + xfail_issue_38713, + "OnnxBackendNodeModelTest.test_momentum_cpu", + "OnnxBackendNodeModelTest.test_nesterov_momentum_cpu", + "OnnxBackendNodeModelTest.test_momentum_multiple_cpu", + ), + ( + xfail_issue_47337, + "OnnxBackendNodeModelTest.test_onehot_without_axis_cpu", + "OnnxBackendNodeModelTest.test_onehot_with_negative_axis_cpu", + "OnnxBackendNodeModelTest.test_onehot_with_axis_cpu", + "OnnxBackendNodeModelTest.test_onehot_negative_indices_cpu", + ), + ( + xfail_issue_33488, + "OnnxBackendNodeModelTest.test_maxunpool_export_with_output_shape_cpu", + "OnnxBackendNodeModelTest.test_maxunpool_export_without_output_shape_cpu", + ), + ( + xfail_issue_33589, + "OnnxBackendNodeModelTest.test_isnan_cpu", + "OnnxBackendNodeModelTest.test_isinf_positive_cpu", + "OnnxBackendNodeModelTest.test_isinf_negative_cpu", + "OnnxBackendNodeModelTest.test_isinf_cpu", + ), + ( + xfail_issue_33535, + "OnnxBackendNodeModelTest.test_dynamicquantizelinear_min_adjusted_cpu", + "OnnxBackendNodeModelTest.test_dynamicquantizelinear_cpu", + "OnnxBackendNodeModelTest.test_dynamicquantizelinear_max_adjusted_cpu", + ), + ( + xfail_issue_38722, + "OnnxBackendNodeModelTest.test_matmulinteger_cpu", + "OnnxBackendNodeModelTest.test_qlinearmatmul_2D_cpu", + "OnnxBackendNodeModelTest.test_qlinearmatmul_3D_cpu", + ), + (xfail_issue_38723, "OnnxBackendNodeModelTest.test_qlinearconv_cpu"), + (xfail_issue_38724, "OnnxBackendNodeModelTest.test_resize_tf_crop_and_resize_cpu"), + ( + xfail_issue_33606, + "OnnxBackendNodeModelTest.test_det_2d_cpu", + "OnnxBackendNodeModelTest.test_det_nd_cpu", + ), + ( + xfail_issue_38732, + "OnnxBackendNodeModelTest.test_convinteger_without_padding_cpu", + "OnnxBackendNodeModelTest.test_convinteger_with_padding_cpu", + "OnnxBackendNodeModelTest.test_basic_convinteger_cpu", + ), + ( + xfail_issue_38734, + "OnnxBackendNodeModelTest.test_adam_multiple_cpu", + "OnnxBackendNodeModelTest.test_adam_cpu", + ), + ( + xfail_issue_38735, + "OnnxBackendNodeModelTest.test_adagrad_multiple_cpu", + "OnnxBackendNodeModelTest.test_adagrad_cpu", + ), + ( + xfail_issue_48052, + "OnnxBackendNodeModelTest.test_training_dropout_cpu", + "OnnxBackendNodeModelTest.test_training_dropout_mask_cpu", + "OnnxBackendNodeModelTest.test_training_dropout_default_cpu", + "OnnxBackendNodeModelTest.test_training_dropout_zero_ratio_cpu", + "OnnxBackendNodeModelTest.test_training_dropout_default_mask_cpu", + "OnnxBackendNodeModelTest.test_training_dropout_zero_ratio_mask_cpu", + ), + ( + xfail_issue_45180, + "OnnxBackendNodeModelTest.test_reduce_sum_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_negative_axes_keepdims_example_cpu", + ), + ( + xfail_issue_43742, + "OnnxBackendNodeModelTest.test_if_cpu", + "OnnxBackendNodeModelTest.test_if_seq_cpu", + ), + ( + xfail_issue_44848, + "OnnxBackendNodeModelTest.test_range_float_type_positive_delta_cpu", + "OnnxBackendNodeModelTest.test_range_int32_type_negative_delta_cpu", + ), + ( + xfail_issue_44851, + "OnnxBackendNodeModelTest.test_expand_dim_changed_cpu", + "OnnxBackendNodeModelTest.test_expand_dim_unchanged_cpu", + "OnnxBackendSimpleModelTest.test_expand_shape_model1_cpu", + "OnnxBackendSimpleModelTest.test_expand_shape_model2_cpu", + "OnnxBackendSimpleModelTest.test_expand_shape_model3_cpu", + "OnnxBackendSimpleModelTest.test_expand_shape_model4_cpu", + ), + ( + xfail_issue_44854, + "OnnxBackendNodeModelTest.test_split_variable_parts_1d_cpu", + "OnnxBackendNodeModelTest.test_split_variable_parts_2d_cpu", + "OnnxBackendNodeModelTest.test_split_variable_parts_default_axis_cpu", + ), + ( + xfail_issue_44858, + "OnnxBackendNodeModelTest.test_unsqueeze_axis_0_cpu", + "OnnxBackendNodeModelTest.test_unsqueeze_axis_1_cpu", + "OnnxBackendNodeModelTest.test_unsqueeze_axis_2_cpu", + "OnnxBackendNodeModelTest.test_unsqueeze_negative_axes_cpu", + "OnnxBackendNodeModelTest.test_unsqueeze_three_axes_cpu", + "OnnxBackendNodeModelTest.test_unsqueeze_two_axes_cpu", + "OnnxBackendNodeModelTest.test_unsqueeze_unsorted_axes_cpu", + ), + ( + xfail_issue_44956, + "OnnxBackendNodeModelTest.test_loop11_cpu", + "OnnxBackendNodeModelTest.test_range_int32_type_negative_delta_expanded_cpu", + "OnnxBackendNodeModelTest.test_range_float_type_positive_delta_expanded_cpu", + ), + ( + xfail_issue_44957, + "OnnxBackendNodeModelTest.test_compress_0_cpu", + "OnnxBackendNodeModelTest.test_compress_1_cpu", + "OnnxBackendNodeModelTest.test_compress_default_axis_cpu", + "OnnxBackendNodeModelTest.test_compress_negative_axis_cpu", + "OnnxBackendNodeModelTest.test_nonzero_example_cpu", + ), + (xfail_issue_44958, "OnnxBackendNodeModelTest.test_upsample_nearest_cpu"), + ( + xfail_issue_44965, + "OnnxBackendNodeModelTest.test_loop13_seq_cpu", + "OnnxBackendNodeModelTest.test_sequence_insert_at_back_cpu", + "OnnxBackendNodeModelTest.test_sequence_insert_at_front_cpu", + ), + ( + xfail_issue_44968, + "OnnxBackendNodeModelTest.test_squeeze_cpu", + "OnnxBackendNodeModelTest.test_squeeze_negative_axes_cpu", + ), + ( + xfail_issue_44976, + "OnnxBackendNodeModelTest.test_quantizelinear_axis_cpu", + "OnnxBackendNodeModelTest.test_dynamicquantizelinear_min_adjusted_expanded_cpu", + "OnnxBackendNodeModelTest.test_dynamicquantizelinear_expanded_cpu", + "OnnxBackendNodeModelTest.test_dynamicquantizelinear_max_adjusted_expanded_cpu", + "OnnxBackendNodeModelTest.test_quantizelinear_cpu", + ), + ( + xfail_issue_33593, + "OnnxBackendNodeModelTest.test_maxpool_with_argmax_2d_precomputed_strides_cpu", + "OnnxBackendNodeModelTest.test_maxpool_with_argmax_2d_precomputed_pads_cpu", + ), + (xfail_issue_58033, "OnnxBackendNodeModelTest.test_einsum_batch_diagonal_cpu"), + ( + xfail_issue_63033, + "OnnxBackendNodeModelTest.test_batchnorm_epsilon_training_mode_cpu", + "OnnxBackendNodeModelTest.test_batchnorm_example_training_mode_cpu", + ), + (xfail_issue_63036, "OnnxBackendNodeModelTest.test_convtranspose_autopad_same_cpu"), + ( + xfail_issue_63039, + "OnnxBackendNodeModelTest.test_div_uint8_cpu", + "OnnxBackendNodeModelTest.test_mul_uint8_cpu", + "OnnxBackendNodeModelTest.test_sub_uint8_cpu", + ), + ( + xfail_issue_63043, + "OnnxBackendNodeModelTest.test_gru_batchwise_cpu", + "OnnxBackendNodeModelTest.test_lstm_batchwise_cpu", + "OnnxBackendNodeModelTest.test_simple_rnn_batchwise_cpu", + ), + ( + xfail_issue_38710, + "OnnxBackendNodeModelTest.test_reshape_allowzero_reordered_cpu", + ), + ( + xfail_issue_63044, + "OnnxBackendNodeModelTest.test_tril_cpu", + "OnnxBackendNodeModelTest.test_tril_neg_cpu", + "OnnxBackendNodeModelTest.test_tril_one_row_neg_cpu", + "OnnxBackendNodeModelTest.test_tril_out_neg_cpu", + "OnnxBackendNodeModelTest.test_tril_out_pos_cpu", + "OnnxBackendNodeModelTest.test_tril_pos_cpu", + "OnnxBackendNodeModelTest.test_tril_square_cpu", + "OnnxBackendNodeModelTest.test_tril_square_neg_cpu", + "OnnxBackendNodeModelTest.test_tril_zero_cpu", + "OnnxBackendNodeModelTest.test_triu_cpu", + "OnnxBackendNodeModelTest.test_triu_neg_cpu", + "OnnxBackendNodeModelTest.test_triu_one_row_cpu", + "OnnxBackendNodeModelTest.test_triu_out_neg_out_cpu", + "OnnxBackendNodeModelTest.test_triu_out_pos_cpu", + "OnnxBackendNodeModelTest.test_triu_pos_cpu", + "OnnxBackendNodeModelTest.test_triu_square_cpu", + "OnnxBackendNodeModelTest.test_triu_square_neg_cpu", + "OnnxBackendNodeModelTest.test_triu_zero_cpu", + ), + ( + xfail_issue_63045, + "OnnxBackendPyTorchConvertedModelTest.test_MaxPool1d_stride_padding_dilation_cpu", + "OnnxBackendPyTorchConvertedModelTest.test_MaxPool2d_stride_padding_dilation_cpu", + ), + ( + skip_rng_tests, + "OnnxBackendNodeModelTest.test_bernoulli_cpu", + "OnnxBackendNodeModelTest.test_bernoulli_double_cpu", + "OnnxBackendNodeModelTest.test_bernoulli_double_expanded_cpu", + "OnnxBackendNodeModelTest.test_bernoulli_expanded_cpu", + "OnnxBackendNodeModelTest.test_bernoulli_seed_cpu", + "OnnxBackendNodeModelTest.test_bernoulli_seed_expanded_cpu", + ), + ( + xfail_issue_63136, + "OnnxBackendNodeModelTest.test_castlike_BFLOAT16_to_FLOAT_cpu", + "OnnxBackendNodeModelTest.test_castlike_DOUBLE_to_FLOAT16_cpu", + "OnnxBackendNodeModelTest.test_castlike_DOUBLE_to_FLOAT_cpu", + "OnnxBackendNodeModelTest.test_castlike_FLOAT16_to_DOUBLE_cpu", + "OnnxBackendNodeModelTest.test_castlike_FLOAT16_to_FLOAT_cpu", + "OnnxBackendNodeModelTest.test_castlike_FLOAT_to_BFLOAT16_cpu", + "OnnxBackendNodeModelTest.test_castlike_FLOAT_to_DOUBLE_cpu", + "OnnxBackendNodeModelTest.test_castlike_FLOAT_to_FLOAT16_cpu", + ), + ( + xfail_issue_63137, + "OnnxBackendNodeModelTest.test_optional_get_element_cpu", + "OnnxBackendNodeModelTest.test_optional_get_element_sequence_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_empty_cpu", + ), + ( + xfail_issue_63138, + "OnnxBackendNodeModelTest.test_shape_end_1_cpu", + "OnnxBackendNodeModelTest.test_shape_end_negative_1_cpu", + "OnnxBackendNodeModelTest.test_shape_start_1_cpu", + "OnnxBackendNodeModelTest.test_shape_start_1_end_2_cpu", + "OnnxBackendNodeModelTest.test_shape_start_1_end_negative_1_cpu", + "OnnxBackendNodeModelTest.test_shape_start_negative_1_cpu", + ), ] for test_group in tests_expected_to_fail: diff --git a/ngraph/python/tests/test_onnx/test_zoo_models.py b/ngraph/python/tests/test_onnx/test_zoo_models.py index 720659a19f3..dd253460044 100644 --- a/ngraph/python/tests/test_onnx/test_zoo_models.py +++ b/ngraph/python/tests/test_onnx/test_zoo_models.py @@ -1,11 +1,9 @@ # Copyright (C) 2018-2021 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import pytest import tests from operator import itemgetter from pathlib import Path -import os from typing import Sequence, Any import numpy as np @@ -134,7 +132,7 @@ for path in Path(MODELS_ROOT_DIR).rglob("*.onnx"): zoo_models.append(model) if len(zoo_models) > 0: - sorted(zoo_models, key=itemgetter("model_name")) + zoo_models = sorted(zoo_models, key=itemgetter("model_name")) # Set backend device name to be used instead of hardcoded by ONNX BackendTest class ones. OpenVinoOnnxBackend.backend_name = tests.BACKEND_NAME diff --git a/ngraph/python/tests/test_onnx/utils/model_importer.py b/ngraph/python/tests/test_onnx/utils/model_importer.py index 44e09e4860f..4979a5cf5bd 100644 --- a/ngraph/python/tests/test_onnx/utils/model_importer.py +++ b/ngraph/python/tests/test_onnx/utils/model_importer.py @@ -33,6 +33,7 @@ class ModelImportRunner(onnx.backend.test.BackendTest): self._include_patterns = set() # type: Set[Pattern[Text]] self._exclude_patterns = set() # type: Set[Pattern[Text]] self._test_items = defaultdict(dict) # type: Dict[Text, Dict[Text, TestItem]] + self._xfail_patterns = set() # type: Set[Pattern[Text]] for model in models: test_name = "test{}".format(model["model_name"]) \ diff --git a/ngraph/test/CMakeLists.txt b/ngraph/test/CMakeLists.txt index 7eed6dd82b5..a851e199d73 100644 --- a/ngraph/test/CMakeLists.txt +++ b/ngraph/test/CMakeLists.txt @@ -8,11 +8,11 @@ set(NGRAPH_ONNX_NAMESPACE ngraph_onnx) add_subdirectory(runtime) if(NOT NGRAPH_UNIT_TEST_ENABLE) - message(STATUS "unit tests disabled") + message(STATUS "nGraph unit tests disabled") return() endif() -message(STATUS "unit tests enabled") +message(STATUS "nGraph unit tests enabled") if(LINUX) set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) diff --git a/ngraph/test/requirements_test_onnx.txt b/ngraph/test/requirements_test_onnx.txt index 4e7803c987b..86e1f720c0d 100644 --- a/ngraph/test/requirements_test_onnx.txt +++ b/ngraph/test/requirements_test_onnx.txt @@ -1,4 +1,4 @@ # ONNX - generate test models docopt~=0.6.2 -onnx~=1.9.0 +onnx~=1.10.0 protobuf>=3.9 \ No newline at end of file diff --git a/thirdparty/onnx/onnx b/thirdparty/onnx/onnx index 685898bc6f3..1089b9e8045 160000 --- a/thirdparty/onnx/onnx +++ b/thirdparty/onnx/onnx @@ -1 +1 @@ -Subproject commit 685898bc6f3fd6eb16ba129ea990f96330537e38 +Subproject commit 1089b9e8045a3a2882d7bb6a1dbaeaf9cae131da