IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move ONNX FE tests into ONNX FE directory (#18259)

Browse Source
This commit is contained in:
Przemyslaw Wysocki authored and GitHub committed 2023-08-01 10:41:05 +02:00
1 parent 6cc8661efe
commit 4a6282a6f5
42 files changed
+286 -243
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.ci/azure/linux.yml
+8 -4
View File
@@ -445,14 +445,18 @@ jobs:
--ignore=$(INSTALL_TEST_DIR)/pyngraph/tests/test_onnx/test_backend.py
displayName: 'nGraph and IE Python Bindings Tests'
# Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time
- script: |
python3 -m pytest -sv $(INSTALL_TEST_DIR)/pyopenvino $(PYTHON_STATIC_ARGS) \
--junitxml=$(INSTALL_TEST_DIR)/TEST-Pyngraph.xml \
--ignore=$(INSTALL_TEST_DIR)/pyopenvino/tests/test_utils/test_utils.py \
--ignore=$(INSTALL_TEST_DIR)/pyopenvino/tests/test_onnx/test_zoo_models.py \
--ignore=$(INSTALL_TEST_DIR)/pyopenvino/tests/test_onnx/test_backend.py
--ignore=$(INSTALL_TEST_DIR)/pyopenvino/tests/test_utils/test_utils.py
displayName: 'Python API 2.0 Tests'
# Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time
- script: |
python3 -m pytest -sv $(REPO_DIR)/src/frontends/onnx/tests $(PYTHON_STATIC_ARGS) \
--ignore=$(REPO_DIR)/src/frontends/onnx/tests/test_python/test_zoo_models.py \
--ignore=$(REPO_DIR)/src/frontends/onnx/tests/test_python/test_backend.py -v
displayName: 'ONNX Frontend Python Tests'
- script: |
python3 -m pytest -s $(INSTALL_TEST_DIR)/mo/unit_tests --junitxml=$(INSTALL_TEST_DIR)/TEST-ModelOptimizer.xml
.ci/azure/linux_debian.yml
+11 -4
View File
@@ -351,13 +351,10 @@ jobs:
LD_LIBRARY_PATH: $(INSTALL_TEST_DIR)
displayName: 'nGraph and IE Python Bindings Tests'
# Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time
- script: |
/usr/bin/python3 -m pytest -s $(INSTALL_TEST_DIR)/pyopenvino \
--junitxml=$(INSTALL_TEST_DIR)/TEST-Pyngraph.xml \
--ignore=$(INSTALL_TEST_DIR)/pyopenvino/tests/test_utils/test_utils.py \
--ignore=$(INSTALL_TEST_DIR)/pyopenvino/tests/test_onnx/test_zoo_models.py \
--ignore=$(INSTALL_TEST_DIR)/pyopenvino/tests/test_onnx/test_backend.py -v
--ignore=$(INSTALL_TEST_DIR)/pyopenvino/tests/test_utils/test_utils.py -v
env:
# Required by python imports to load requires libraries
# - tests install dir for mock_py
@@ -366,6 +363,16 @@ jobs:
PYTHONPATH: $(INSTALL_TEST_DIR)
displayName: 'Python API 2.0 Tests'
# Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time
- script: |
/usr/bin/python3 -m pytest -s $(REPO_DIR)/src/frontends/onnx/tests \
--ignore=$(REPO_DIR)/src/frontends/onnx/tests/test_python/test_zoo_models.py \
--ignore=$(REPO_DIR)/src/frontends/onnx/tests/test_python/test_backend.py -v
env:
LD_LIBRARY_PATH: $(INSTALL_TEST_DIR)
PYTHONPATH: $(INSTALL_TEST_DIR)
displayName: 'ONNX Frontend Python Tests'
- script: |
# TODO: fix 'No mock frontend API available'
$(TEST_VENV)/bin/python3 -m pip install openvino-dev --find-links=$(INSTALL_DIR)/tools
.ci/azure/linux_ngraph_onnx.yml
+1 -1
View File
@@ -102,7 +102,7 @@ jobs:
displayName: 'Install dependencies'
- script:
src/bindings/python/tests/test_onnx/model_zoo_preprocess.sh -d $(MODELS_DIR)/models_data -o -s "$(ONNX_MODEL_ZOO_SHA)"
src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh -d $(MODELS_DIR)/models_data -o -s "$(ONNX_MODEL_ZOO_SHA)"
displayName: 'Update models'
condition: ne(variables['BUILD_TYPE'], 'Debug')
src/bindings/python/constraints.txt
+3 -3
View File
@@ -1,7 +1,5 @@
# used in multiple components
onnx==1.13.1 # Python bindings, ONNX Frontend
numpy>=1.16.6,<1.26 # Python bindings, frontends
protobuf>=3.18.1,<4.0.0 # Python bindings, frontends
# pytest
pytest>=5.0,<7.4
@@ -19,4 +17,6 @@ wheel>=0.38.1
docopt~=0.6.2
paddlepaddle==2.4.2
tensorflow>=1.15.5,<2.13.0
six~=1.16.0
six~=1.16.0
protobuf>=3.18.1,<4.0.0
onnx==1.13.1
src/bindings/python/requirements_test.txt
-2
View File
@@ -1,7 +1,5 @@
-c ./constraints.txt
protobuf
numpy
onnx
bandit
black
flake8
src/bindings/python/setup.cfg
+4 -2
View File
@@ -7,6 +7,7 @@ skip_install=True
deps =
-rrequirements.txt
-rrequirements_test.txt
-r /openvino/src/frontends/onnx/tests/requirements.txt
setenv =
OV_BACKEND = {env:OV_BACKEND:"CPU"}
PYTHONPATH = {env:PYTHONPATH}
@@ -15,12 +16,13 @@ passenv =
http_proxy
https_proxy
commands=
pytest --backend={env:OV_BACKEND} tests -m "not template_plugin" -v -k 'not _cuda' --ignore=tests/test_onnx/test_zoo_models.py --ignore=tests/test_utils
pytest tests -m "not template_plugin" -v -k 'not _cuda' --ignore=tests/test_utils
pytest --backend={env:OV_BACKEND} tests_compatibility/test_ngraph -v -k 'not _cuda' --ignore=tests_compatibility/test_onnx/test_zoo_models.py
pytest --backend={env:OV_BACKEND} /openvino/src/frontends/onnx/tests -v --ignore=/openvino/src/frontends/onnx/tests/tests_python/test_zoo_models.py
[testenv:zoo_models]
commands=
pytest --backend={env:OV_BACKEND} tests/test_onnx/test_zoo_models.py -v -n 4 --forked -k 'not _cuda' --model_zoo_xfail
pytest --backend={env:OV_BACKEND} /openvino/src/frontends/onnx/tests/tests_python/test_zoo_models.py -v -n 4 --forked -k 'not _cuda' --model_zoo_xfail
[testenv:devenv]
envdir = devenv
src/bindings/python/tests/__init__.py
-145
View File
@@ -4,152 +4,7 @@
import pytest
# test.BACKEND_NAME is a configuration variable determining which
# OV backend tests will use. It's set during pytest configuration time.
# See `pytest_configure` hook in `conftest.py` for more details.
BACKEND_NAME = None
# test.MODEL_ZOO_DIR is a configuration variable providing the path
# to the ZOO of ONNX models to test. It's set during pytest configuration time.
# See `pytest_configure` hook in `conftest.py` for more
# details.
MODEL_ZOO_DIR = None
# test.MODEL_ZOO_XFAIL is a configuration variable which enable xfails for model zoo.
MODEL_ZOO_XFAIL = False
def xfail_test(reason="Mark the test as expected to fail", strict=True):
return pytest.mark.xfail(reason=reason, strict=strict)
skip_segfault = pytest.mark.skip(reason="Segmentation fault error")
xfail_dynamic_rank = xfail_test(reason="Dynamic rank")
skip_devtest = pytest.mark.skip(reason="Test might depend on machine, should be run by developers"
"or advanced users for debug/testing purposes.")
skip_need_mock_op = pytest.mark.skip(reason="Test need to be rewritten with mock operation. Issue #101215")
xfail_accuracy = xfail_test(reason="Accuracy")
xfail_issue_69444 = xfail_test(reason="ONNX Resize - AssertionError: Mismatched elements.")
skip_issue_67415 = pytest.mark.skip(reason="RuntimeError: Unsupported data type for when filling blob!")
xfail_issue_67415 = xfail_test(reason="RuntimeError: Unsupported data type for when filling blob!")
xfail_issue_33488 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"MaxUnpool")
skip_issue_38084 = pytest.mark.skip(reason="Aborted (core dumped) Assertion "
"`(layer->get_output_partial_shape(i).is_static())' failed.")
xfail_issue_33596 = xfail_test(reason="RuntimeError: OV does not support different sequence operations: "
"ConcatFromSequence, SequenceConstruct, SequenceAt, SplitToSequence, "
"SequenceEmpty, SequenceInsert, SequenceErase, SequenceLength ")
xfail_issue_33606 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"Det")
xfail_issue_33651 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"TfIdfVectorizer")
xfail_issue_33581 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"GatherElements")
xfail_issue_90649 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations:"
"BlackmanWindow, DFT, HammingWindow, HannWindow, LayerNormalization, "
"MelWeightMatrix, SequenceMap, STFT")
xfail_issue_35923 = xfail_test(reason="RuntimeError: PReLU without weights is not supported")
xfail_issue_35927 = xfail_test(reason="RuntimeError: B has zero dimension that is not allowable")
xfail_issue_38091 = xfail_test(reason="AssertionError: Mismatched elements")
xfail_issue_38699 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"ai.onnx.preview.training.Gradient")
xfail_issue_38701 = xfail_test(reason="RuntimeError: unsupported element type: STRING")
xfail_issue_38706 = xfail_test(reason="RuntimeError: output_3.0 has zero dimension which is not allowed")
xfail_issue_38708 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(Slice): y>': "
"Axes input must be constant")
xfail_issue_99949 = xfail_test(reason="Bitwise operators are not supported")
xfail_issue_99950 = xfail_test(reason="CenterCropPad func is not supported")
xfail_issue_99952 = xfail_test(reason="Col2Im operator is not supported")
xfail_issue_99954 = xfail_test(reason="Constant Pad - RuntimeError: Shape inference of Reference node with name y failed")
xfail_issue_99955 = xfail_test(reason="GroupNorm is not supported")
xfail_issue_99957 = xfail_test(reason="LayerNorm - RuntimeError: While validating node '<Node(Reshape): Mean>'")
xfail_issue_99958 = xfail_test(reason="LogSoftmax - Results mismatch")
xfail_issue_99959 = xfail_test(reason="Mish function is not supported")
xfail_issue_99960 = xfail_test(reason="MVN - Results mismatch")
xfail_issue_99961 = xfail_test(reason="Optional has/get element operators are not supported)'")
xfail_issue_99962 = pytest.mark.skip(reason="ReduceL1/L2 - Unrecognized attribute: axes for operator ReduceL1/L2")
xfail_issue_99968 = xfail_test(reason="ReduceL1/L2 - Results mismatch or unsupported ReduceSum with "
"dynamic rank by CPU plugin")
xfail_issue_99969 = xfail_test(reason="Resize - Results mismatch / "
"RuntimeError: While validating ONNX node '<Node(Resize): Y>' / "
"RuntimeError: Check '(false)' failed at onnx/frontend/src/op/resize.cpp")
xfail_issue_99970 = xfail_test(reason="Scatter and ScatterND - RuntimeError: Check '(reduction == none)' failed at "
"src/frontends/onnx/frontend/src/op/scatter_elements.cpp OR at "
"src/frontends/onnx/frontend/src/op/scatter_nd")
xfail_issue_99972 = xfail_test(reason="Softmax - Results mismatch")
xfail_issue_99973 = xfail_test(reason="Split - RuntimeError: While validating ONNX node "
"'<Node(Split): output_1, output_2, output_3, output_4>'")
xfail_issue_38710 = xfail_test(reason="RuntimeError: data has zero dimension which is not allowed")
xfail_issue_38713 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"ai.onnx.preview.training.Momentum")
xfail_issue_38724 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(Resize): Y>': "
"tf_crop_and_resize - this type of coordinate transformation mode "
"is not supported. Choose one of the following modes: "
"tf_half_pixel_for_nn, asymmetric, align_corners, pytorch_half_pixel, "
"half_pixel")
xfail_issue_38725 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(Loop): "
"value info has no element type specified")
xfail_issue_38734 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"ai.onnx.preview.training.Adam")
xfail_issue_38735 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"ai.onnx.preview.training.Adagrad")
xfail_issue_48052 = xfail_test(reason="Dropout op is not supported in traning mode")
xfail_issue_45180 = xfail_test(reason="RuntimeError: Unsupported dynamic op: ReduceSum")
xfail_issue_44851 = xfail_test(reason="Expected: Unsupported dynamic op: Broadcast")
xfail_issue_44858 = xfail_test(reason="Expected: Unsupported dynamic op: Unsqueeze")
xfail_issue_44957 = xfail_test(reason="Expected: Unsupported dynamic op: NonZero")
xfail_issue_44958 = xfail_test(reason="Expected: Unsupported dynamic op: Interpolate")
xfail_issue_44965 = xfail_test(reason="Expected: RuntimeError: value info has no element")
xfail_issue_47323 = xfail_test(reason="RuntimeError: The plugin does not support FP64")
xfail_issue_73538 = xfail_test(reason="OneHot: Unsupported negative indices, "
"AssertionError: Mismatched elements.")
# Model MSFT issues:
xfail_issue_37957 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"com.microsoft.CropAndResize, com.microsoft.GatherND, "
"com.microsoft.Pad, com.microsoft.Range")
xfail_issue_39669 = xfail_test(reason="AssertionError: This model has no test data")
xfail_issue_36534 = xfail_test(reason="RuntimeError: node input index is out of range")
xfail_issue_36536 = xfail_test(reason="RuntimeError: can't protect")
xfail_issue_36538 = xfail_test(reason="RuntimeError: Check 'PartialShape::broadcast_merge_into( pshape, "
"node->get_input_partial_shape(i), autob)' failed at "
"/openvino/ngraph/src/ngraph/op/util/elementwise_args.cpp:48:")
xfail_issue_39658 = xfail_test(reason="RuntimeError: Tile operation has a form that is not supported."
" z should be converted to TileIE operation.")
xfail_issue_37973 = xfail_test(reason="TF Inception V2 - AssertionError: zoo models results mismatch")
xfail_issue_47495 = xfail_test(reason="BertSquad-10 from MSFT - AssertionError: zoo models results mismatch")
xfail_issue_48145 = xfail_test(reason="BertSquad-8 - AssertionError: Items are not equal: ACTUAL: 4 "
"DESIRED: 3")
xfail_issue_48190 = xfail_test(reason="RobertaBase-11 - AssertionError: Items are not equal: "
"ACTUAL: dtype('float64') DESIRED: dtype('float32')")
xfail_issue_49752 = xfail_test(reason="RuntimeError: Unsupported dynamic ops: v1::Pad")
xfail_issue_49753 = xfail_test(reason="RuntimeError: Unsupported dynamic ops: v1::StridedSlice")
xfail_issue_49754 = xfail_test(reason="RuntimeError: Unsupported dynamic ops: v1::TopKIE")
xfail_issue_52463 = xfail_test(reason="test_operator_add_size1_singleton_broadcast_cpu - "
"Not equal to tolerance")
xfail_issue_58033 = xfail_test(reason="Einsum operation misses support for complex ellipsis equations")
xfail_issue_58676 = xfail_test(reason="AssertionError: Not equal to tolerance rtol=0.001, atol=1e-07")
xfail_issue_onnx_models_140 = xfail_test(reason="https://github.com/onnx/models/issues/140")
xfail_issue_63033 = xfail_test(reason="BatchNormalization: Training mode is not supported")
xfail_issue_63036 = xfail_test(reason="Changes in ConvTranspose padding")
xfail_issue_63043 = xfail_test(reason="Recurrent node expects constants as W, R, B inputs.")
skip_rng_tests = pytest.mark.skip(reason="Tests use random number generator with no seed.")
xfail_issue_63137 = xfail_test(reason="Unsupported operations: OptionalHasElement, OptionalGetElement")
xfail_issue_63138 = xfail_test(reason="Missing ONNX Shape-15 support")
xfail_issue_68212 = xfail_test(reason="Unsupported reading model with bytes streams")
xfail_issue_78843 = xfail_test(reason="Missing reference output files for ssd mobilenet models")
xfail_issue_81976 = xfail_test(reason="RuntimeError: z node not found in OV cache")
xfail_issue_82038 = xfail_test(reason="ScatterElements, ScatterND, AssertionError: Result mismatch")
xfail_issue_82039 = xfail_test(reason="Unsupported data type Optional, RuntimeError: [ NOT_IMPLEMENTED ] "
"CPU plugin: Input image format UNSPECIFIED is not supported yet...")
xfail_issue_86911 = xfail_test(reason="LSTM_Seq_len_unpacked - AssertionError: zoo models results mismatch")
xfail_issue_91151 = xfail_test(reason="RuntimeError: model input (shape={3,4}) and blob (shape=(1)) are incompatible")
xfail_issue_91490 = xfail_test(reason="y has zero dimension which is not allowed")
xfail_issue_101965 = xfail_test(reason="Mismatch with numpy-based expected results.")
xfail_issue_113506 = xfail_test(reason="Unsupported operation of type: LSTMSequence Node expects 7 inputs. Actual: 8")
src/bindings/python/tests/conftest.py
-58
View File
@@ -5,9 +5,6 @@
import os
import pytest
import tests
from pathlib import Path
from sys import platform
from openvino.runtime import Core
@@ -85,35 +82,7 @@ def model_onnx_path():
return test_onnx
def _get_default_model_zoo_dir():
return Path(os.getenv("ONNX_HOME", Path.home() / ".onnx/model_zoo"))
def pytest_addoption(parser):
parser.addoption(
"--backend",
default="CPU",
choices=["CPU", "GPU", "GNA", "HETERO", "TEMPLATE"],
help="Select target device",
)
parser.addoption(
"--model_zoo_dir",
default=_get_default_model_zoo_dir(),
type=str,
help="location of the model zoo",
)
parser.addoption(
"--model_zoo_xfail",
action="store_true",
help="treat model zoo known issues as xfails instead of failures",
)
def pytest_configure(config):
backend_name = config.getvalue("backend")
tests.BACKEND_NAME = backend_name
tests.MODEL_ZOO_DIR = Path(config.getvalue("model_zoo_dir"))
tests.MODEL_ZOO_XFAIL = config.getvalue("model_zoo_xfail")
# register additional markers
config.addinivalue_line("markers", "skip_on_cpu: Skip test on CPU")
@@ -126,33 +95,6 @@ def pytest_configure(config):
config.addinivalue_line("markers", "dynamic_library: Runs tests only in dynamic libraries case")
def pytest_collection_modifyitems(config, items):
backend_name = config.getvalue("backend")
tests.MODEL_ZOO_DIR = Path(config.getvalue("model_zoo_dir"))
tests.MODEL_ZOO_XFAIL = config.getvalue("model_zoo_xfail")
keywords = {
"CPU": "skip_on_cpu",
"GPU": "skip_on_gpu",
"GNA": "skip_on_gna",
"HETERO": "skip_on_hetero",
"TEMPLATE": "skip_on_template",
}
skip_markers = {
"CPU": pytest.mark.skip(reason="Skipping test on the CPU backend."),
"GPU": pytest.mark.skip(reason="Skipping test on the GPU backend."),
"GNA": pytest.mark.skip(reason="Skipping test on the GNA backend."),
"HETERO": pytest.mark.skip(reason="Skipping test on the HETERO backend."),
"TEMPLATE": pytest.mark.skip(reason="Skipping test on the TEMPLATE backend."),
}
for item in items:
skip_this_backend = keywords[backend_name]
if skip_this_backend in item.keywords:
item.add_marker(skip_markers[backend_name])
@pytest.fixture(scope="session")
def device():
return os.environ.get("TEST_DEVICE") if os.environ.get("TEST_DEVICE") else "CPU"
src/bindings/python/tests/test_frontend/__init__.py
-3
View File
@@ -1,3 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
src/frontends/onnx/tests/__init__.py
+152
View File
@@ -0,0 +1,152 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2023 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import pytest
# test.BACKEND_NAME is a configuration variable determining which
# OV backend tests will use. It's set during pytest configuration time.
# See `pytest_configure` hook in `conftest.py` for more details.
BACKEND_NAME = None
# test.MODEL_ZOO_DIR is a configuration variable providing the path
# to the ZOO of ONNX models to test. It's set during pytest configuration time.
# See `pytest_configure` hook in `conftest.py` for more
# details.
MODEL_ZOO_DIR = None
# test.MODEL_ZOO_XFAIL is a configuration variable which enable xfails for model zoo.
MODEL_ZOO_XFAIL = False
def xfail_test(reason="Mark the test as expected to fail", strict=True):
return pytest.mark.xfail(reason=reason, strict=strict)
skip_segfault = pytest.mark.skip(reason="Segmentation fault error")
xfail_dynamic_rank = xfail_test(reason="Dynamic rank")
xfail_accuracy = xfail_test(reason="Accuracy")
xfail_issue_69444 = xfail_test(reason="ONNX Resize - AssertionError: Mismatched elements.")
skip_issue_67415 = pytest.mark.skip(reason="RuntimeError: Unsupported data type for when filling blob!")
xfail_issue_67415 = xfail_test(reason="RuntimeError: Unsupported data type for when filling blob!")
xfail_issue_33488 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"MaxUnpool")
skip_issue_38084 = pytest.mark.skip(reason="Aborted (core dumped) Assertion "
"`(layer->get_output_partial_shape(i).is_static())' failed.")
xfail_issue_33596 = xfail_test(reason="RuntimeError: OV does not support different sequence operations: "
"ConcatFromSequence, SequenceConstruct, SequenceAt, SplitToSequence, "
"SequenceEmpty, SequenceInsert, SequenceErase, SequenceLength ")
xfail_issue_33606 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"Det")
xfail_issue_33651 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"TfIdfVectorizer")
xfail_issue_33581 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"GatherElements")
xfail_issue_90649 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations:"
"BlackmanWindow, DFT, HammingWindow, HannWindow, LayerNormalization, "
"MelWeightMatrix, SequenceMap, STFT")
xfail_issue_35923 = xfail_test(reason="RuntimeError: PReLU without weights is not supported")
xfail_issue_35927 = xfail_test(reason="RuntimeError: B has zero dimension that is not allowable")
xfail_issue_38091 = xfail_test(reason="AssertionError: Mismatched elements")
xfail_issue_38699 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"ai.onnx.preview.training.Gradient")
xfail_issue_38701 = xfail_test(reason="RuntimeError: unsupported element type: STRING")
xfail_issue_38706 = xfail_test(reason="RuntimeError: output_3.0 has zero dimension which is not allowed")
xfail_issue_38708 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(Slice): y>': "
"Axes input must be constant")
xfail_issue_99949 = xfail_test(reason="Bitwise operators are not supported")
xfail_issue_99950 = xfail_test(reason="CenterCropPad func is not supported")
xfail_issue_99952 = xfail_test(reason="Col2Im operator is not supported")
xfail_issue_99954 = xfail_test(reason="Constant Pad - RuntimeError: Shape inference of Reference node with name y failed")
xfail_issue_99955 = xfail_test(reason="GroupNorm is not supported")
xfail_issue_99957 = xfail_test(reason="LayerNorm - RuntimeError: While validating node '<Node(Reshape): Mean>'")
xfail_issue_99958 = xfail_test(reason="LogSoftmax - Results mismatch")
xfail_issue_99959 = xfail_test(reason="Mish function is not supported")
xfail_issue_99960 = xfail_test(reason="MVN - Results mismatch")
xfail_issue_99961 = xfail_test(reason="Optional has/get element operators are not supported)'")
xfail_issue_99962 = pytest.mark.skip(reason="ReduceL1/L2 - Unrecognized attribute: axes for operator ReduceL1/L2")
xfail_issue_99968 = xfail_test(reason="ReduceL1/L2 - Results mismatch or unsupported ReduceSum with "
"dynamic rank by CPU plugin")
xfail_issue_99969 = xfail_test(reason="Resize - Results mismatch / "
"RuntimeError: While validating ONNX node '<Node(Resize): Y>' / "
"RuntimeError: Check '(false)' failed at onnx/frontend/src/op/resize.cpp")
xfail_issue_99970 = xfail_test(reason="Scatter and ScatterND - RuntimeError: Check '(reduction == none)' failed at "
"src/frontends/onnx/frontend/src/op/scatter_elements.cpp OR at "
"src/frontends/onnx/frontend/src/op/scatter_nd")
xfail_issue_99972 = xfail_test(reason="Softmax - Results mismatch")
xfail_issue_99973 = xfail_test(reason="Split - RuntimeError: While validating ONNX node "
"'<Node(Split): output_1, output_2, output_3, output_4>'")
xfail_issue_38710 = xfail_test(reason="RuntimeError: data has zero dimension which is not allowed")
xfail_issue_38713 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"ai.onnx.preview.training.Momentum")
xfail_issue_38724 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(Resize): Y>': "
"tf_crop_and_resize - this type of coordinate transformation mode "
"is not supported. Choose one of the following modes: "
"tf_half_pixel_for_nn, asymmetric, align_corners, pytorch_half_pixel, "
"half_pixel")
xfail_issue_38725 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(Loop): "
"value info has no element type specified")
xfail_issue_38734 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"ai.onnx.preview.training.Adam")
xfail_issue_38735 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"ai.onnx.preview.training.Adagrad")
xfail_issue_48052 = xfail_test(reason="Dropout op is not supported in traning mode")
xfail_issue_45180 = xfail_test(reason="RuntimeError: Unsupported dynamic op: ReduceSum")
xfail_issue_44851 = xfail_test(reason="Expected: Unsupported dynamic op: Broadcast")
xfail_issue_44858 = xfail_test(reason="Expected: Unsupported dynamic op: Unsqueeze")
xfail_issue_44957 = xfail_test(reason="Expected: Unsupported dynamic op: NonZero")
xfail_issue_44958 = xfail_test(reason="Expected: Unsupported dynamic op: Interpolate")
xfail_issue_44965 = xfail_test(reason="Expected: RuntimeError: value info has no element")
xfail_issue_47323 = xfail_test(reason="RuntimeError: The plugin does not support FP64")
xfail_issue_73538 = xfail_test(reason="OneHot: Unsupported negative indices, "
"AssertionError: Mismatched elements.")
# Model MSFT issues:
xfail_issue_37957 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: "
"com.microsoft.CropAndResize, com.microsoft.GatherND, "
"com.microsoft.Pad, com.microsoft.Range")
xfail_issue_39669 = xfail_test(reason="AssertionError: This model has no test data")
xfail_issue_36534 = xfail_test(reason="RuntimeError: node input index is out of range")
xfail_issue_36536 = xfail_test(reason="RuntimeError: can't protect")
xfail_issue_36538 = xfail_test(reason="RuntimeError: Check 'PartialShape::broadcast_merge_into( pshape, "
"node->get_input_partial_shape(i), autob)' failed at "
"/openvino/ngraph/src/ngraph/op/util/elementwise_args.cpp:48:")
skip_issue_39658 = pytest.mark.skip(reason="RuntimeError: Tile operation has a form that is not supported."
" z should be converted to TileIE operation.")
xfail_issue_37973 = xfail_test(reason="TF Inception V2 - AssertionError: zoo models results mismatch")
xfail_issue_47495 = xfail_test(reason="BertSquad-10 from MSFT - AssertionError: zoo models results mismatch")
xfail_issue_49207 = xfail_test(reason="Model references undeclared parameters")
xfail_issue_48145 = xfail_test(reason="BertSquad-8 - AssertionError: Items are not equal: ACTUAL: 4 "
"DESIRED: 3")
xfail_issue_48190 = xfail_test(reason="RobertaBase-11 - AssertionError: Items are not equal: "
"ACTUAL: dtype('float64') DESIRED: dtype('float32')")
xfail_issue_49752 = xfail_test(reason="RuntimeError: Unsupported dynamic ops: v1::Pad")
xfail_issue_49753 = xfail_test(reason="RuntimeError: Unsupported dynamic ops: v1::StridedSlice")
xfail_issue_49754 = xfail_test(reason="RuntimeError: Unsupported dynamic ops: v1::TopKIE")
xfail_issue_52463 = xfail_test(reason="test_operator_add_size1_singleton_broadcast_cpu - "
"Not equal to tolerance")
xfail_issue_58033 = xfail_test(reason="Einsum operation misses support for complex ellipsis equations")
xfail_issue_58676 = xfail_test(reason="AssertionError: Not equal to tolerance rtol=0.001, atol=1e-07")
xfail_issue_onnx_models_140 = xfail_test(reason="https://github.com/onnx/models/issues/140")
xfail_issue_63033 = xfail_test(reason="BatchNormalization: Training mode is not supported")
xfail_issue_63036 = xfail_test(reason="Changes in ConvTranspose padding")
xfail_issue_63043 = xfail_test(reason="Recurrent node expects constants as W, R, B inputs.")
skip_rng_tests = pytest.mark.skip(reason="Tests use random number generator with no seed.")
xfail_issue_63137 = xfail_test(reason="Unsupported operations: OptionalHasElement, OptionalGetElement")
xfail_issue_63138 = xfail_test(reason="Missing ONNX Shape-15 support")
xfail_issue_68212 = xfail_test(reason="Unsupported reading model with bytes streams")
xfail_issue_78843 = xfail_test(reason="Missing reference output files for ssd mobilenet models")
xfail_issue_81976 = xfail_test(reason="RuntimeError: z node not found in OV cache")
xfail_issue_82038 = xfail_test(reason="ScatterElements, ScatterND, AssertionError: Result mismatch")
xfail_issue_82039 = xfail_test(reason="Unsupported data type Optional, RuntimeError: [ NOT_IMPLEMENTED ] "
"CPU plugin: Input image format UNSPECIFIED is not supported yet...")
xfail_issue_86911 = xfail_test(reason="LSTM_Seq_len_unpacked - AssertionError: zoo models results mismatch")
xfail_issue_91151 = xfail_test(reason="RuntimeError: model input (shape={3,4}) and blob (shape=(1)) are incompatible")
xfail_issue_91490 = xfail_test(reason="y has zero dimension which is not allowed")
xfail_issue_101965 = xfail_test(reason="Mismatch with numpy-based expected results.")
xfail_issue_113506 = xfail_test(reason="Unsupported operation of type: LSTMSequence Node expects 7 inputs. Actual: 8")
src/frontends/onnx/tests/conftest.py
+83
View File
@@ -0,0 +1,83 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2023 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import os
import pytest
import tests
from pathlib import Path
def _get_default_model_zoo_dir():
return Path(os.getenv("ONNX_HOME", Path.home() / ".onnx/model_zoo"))
def pytest_addoption(parser):
parser.addoption(
"--backend",
default="CPU",
choices=["CPU", "GPU", "GNA", "HETERO", "TEMPLATE"],
help="Select target device",
)
parser.addoption(
"--model_zoo_dir",
default=_get_default_model_zoo_dir(),
type=str,
help="location of the model zoo",
)
parser.addoption(
"--model_zoo_xfail",
action="store_true",
help="treat model zoo known issues as xfails instead of failures",
)
def pytest_configure(config):
backend_name = config.getvalue("backend")
tests.BACKEND_NAME = backend_name
tests.MODEL_ZOO_DIR = Path(config.getvalue("model_zoo_dir"))
tests.MODEL_ZOO_XFAIL = config.getvalue("model_zoo_xfail")
# register additional markers
config.addinivalue_line("markers", "skip_on_cpu: Skip test on CPU")
config.addinivalue_line("markers", "skip_on_gpu: Skip test on GPU")
config.addinivalue_line("markers", "skip_on_gna: Skip test on GNA")
config.addinivalue_line("markers", "skip_on_hetero: Skip test on HETERO")
config.addinivalue_line("markers", "skip_on_template: Skip test on TEMPLATE")
config.addinivalue_line("markers", "onnx_coverage: Collect ONNX operator coverage")
config.addinivalue_line("markers", "template_plugin")
config.addinivalue_line("markers", "dynamic_library: Runs tests only in dynamic libraries case")
def pytest_collection_modifyitems(config, items):
backend_name = config.getvalue("backend")
tests.MODEL_ZOO_DIR = Path(config.getvalue("model_zoo_dir"))
tests.MODEL_ZOO_XFAIL = config.getvalue("model_zoo_xfail")
keywords = {
"CPU": "skip_on_cpu",
"GPU": "skip_on_gpu",
"GNA": "skip_on_gna",
"HETERO": "skip_on_hetero",
"TEMPLATE": "skip_on_template",
}
skip_markers = {
"CPU": pytest.mark.skip(reason="Skipping test on the CPU backend."),
"GPU": pytest.mark.skip(reason="Skipping test on the GPU backend."),
"GNA": pytest.mark.skip(reason="Skipping test on the GNA backend."),
"HETERO": pytest.mark.skip(reason="Skipping test on the HETERO backend."),
"TEMPLATE": pytest.mark.skip(reason="Skipping test on the TEMPLATE backend."),
}
for item in items:
skip_this_backend = keywords[backend_name]
if skip_this_backend in item.keywords:
item.add_marker(skip_markers[backend_name])
@pytest.fixture(scope="session")
def device():
return os.environ.get("TEST_DEVICE") if os.environ.get("TEST_DEVICE") else "CPU"
src/frontends/onnx/tests/requirements.txt
+2
View File
@@ -1,5 +1,7 @@
# ONNX - generate test models
-c ../../../bindings/python/constraints.txt
numpy
protobuf
docopt
onnx
pytest
src/bindings/python/tests/runtime.py → src/frontends/onnx/tests/runtime.py
View File
File renamed without changes.
src/bindings/python/tests/test_onnx/__init__.py → src/frontends/onnx/tests/tests_python/__init__.py
View File
File renamed without changes.
src/bindings/python/tests/test_onnx/model_zoo_preprocess.sh → src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh
View File
File renamed without changes.
src/bindings/python/tests/test_onnx/models/add_abc.onnx → src/frontends/onnx/tests/tests_python/models/add_abc.onnx
View File
File renamed without changes.
src/bindings/python/tests/test_onnx/models/data/tensor.data → src/frontends/onnx/tests/tests_python/models/data/tensor.data
View File
File renamed without changes.
src/bindings/python/tests/test_onnx/models/external_data.onnx → src/frontends/onnx/tests/tests_python/models/external_data.onnx
View File
File renamed without changes.
src/bindings/python/tests/test_onnx/test_backend.py → src/frontends/onnx/tests/tests_python/test_backend.py
+4 -3
View File
@@ -22,7 +22,8 @@ from tests import (
xfail_issue_38724,
xfail_issue_38734,
xfail_issue_38735,
xfail_issue_39658,
skip_issue_39658,
skip_issue_39658,
xfail_issue_44858,
xfail_issue_44965,
xfail_issue_45180,
@@ -62,7 +63,7 @@ from tests import (
xfail_issue_101965,
xfail_issue_113506,
)
from tests.test_onnx.utils.onnx_backend import OpenVinoTestBackend
from tests.tests_python.utils.onnx_backend import OpenVinoTestBackend
def expect_fail(test_case_path, xfail): # type: (str) -> None
@@ -113,7 +114,7 @@ globals().update(backend_test.enable_report().test_cases)
tests_expected_to_fail = [
(
xfail_issue_39658,
skip_issue_39658,
"OnnxBackendNodeModelTest.test_tile_cpu",
),
(
src/bindings/python/tests/test_frontend/test_frontend_extension.py → src/frontends/onnx/tests/tests_python/test_frontend_extension.py
View File
File renamed without changes.
src/bindings/python/tests/test_frontend/test_frontend_lib_close.py → src/frontends/onnx/tests/tests_python/test_frontend_lib_close.py
View File
File renamed without changes.
src/bindings/python/tests/test_frontend/test_frontend_onnx.py → src/frontends/onnx/tests/tests_python/test_frontend_onnx.py
View File
File renamed without changes.
src/bindings/python/tests/test_frontend/test_frontend_onnx_editor.py → src/frontends/onnx/tests/tests_python/test_frontend_onnx_editor.py
View File
File renamed without changes.
src/bindings/python/tests/test_frontend/test_frontendmanager.py → src/frontends/onnx/tests/tests_python/test_frontendmanager.py
View File
File renamed without changes.
src/bindings/python/tests/test_onnx/test_onnx_external_data.py → src/frontends/onnx/tests/tests_python/test_onnx_external_data.py
View File
File renamed without changes.
src/bindings/python/tests/test_onnx/test_onnx_import.py → src/frontends/onnx/tests/tests_python/test_onnx_import.py
+1 -1
View File
@@ -10,7 +10,7 @@ from onnx.helper import make_graph, make_model, make_node, make_tensor_value_inf
from openvino.runtime import Core
from tests.runtime import get_runtime
from tests.test_onnx.utils.onnx_helpers import import_onnx_model
from tests.tests_python.utils.onnx_helpers import import_onnx_model
def test_import_onnx_function():
src/bindings/python/tests/test_onnx/test_ops_batchnorm.py → src/frontends/onnx/tests/tests_python/test_ops_batchnorm.py
+1 -1
View File
@@ -5,7 +5,7 @@
import numpy as np
import onnx
from tests.test_onnx.utils import run_node
from tests.tests_python.utils import run_node
def make_batch_norm_node(**node_attributes):
src/bindings/python/tests/test_onnx/test_ops_binary.py → src/frontends/onnx/tests/tests_python/test_ops_binary.py
+1 -1
View File
@@ -7,7 +7,7 @@ import onnx
import pytest
from onnx.helper import make_graph, make_model, make_tensor_value_info
from tests.test_onnx.utils import run_model
from tests.tests_python.utils import run_model
def import_and_compute(op_type, input_data_left, input_data_right, opset=7, **node_attributes):
src/bindings/python/tests/test_onnx/test_ops_convpool.py → src/frontends/onnx/tests/tests_python/test_ops_convpool.py
+1 -1
View File
@@ -9,7 +9,7 @@ from onnx.helper import make_graph, make_model, make_node, make_tensor_value_inf
from onnx.onnx_cpp2py_export.checker import ValidationError
from tests.runtime import get_runtime
from tests.test_onnx.utils import get_node_model, import_onnx_model, run_model, run_node
from tests.tests_python.utils import get_node_model, import_onnx_model, run_model, run_node
@pytest.fixture()
src/bindings/python/tests/test_onnx/test_ops_logical.py → src/frontends/onnx/tests/tests_python/test_ops_logical.py
+1 -1
View File
@@ -6,7 +6,7 @@ import numpy as np
import onnx
import pytest
from tests.test_onnx.utils import run_node
from tests.tests_python.utils import run_node
@pytest.mark.parametrize(
src/bindings/python/tests/test_onnx/test_ops_matmul.py → src/frontends/onnx/tests/tests_python/test_ops_matmul.py
+1 -1
View File
@@ -8,7 +8,7 @@ from onnx.helper import make_graph, make_model, make_node, make_tensor_value_inf
import pytest
from tests.runtime import get_runtime
from tests.test_onnx.utils import import_onnx_model
from tests.tests_python.utils import import_onnx_model
def make_onnx_model_for_matmul_op(input_left, input_right):
src/bindings/python/tests/test_onnx/test_ops_nonlinear.py → src/frontends/onnx/tests/tests_python/test_ops_nonlinear.py
+1 -1
View File
@@ -6,7 +6,7 @@ import numpy as np
import onnx
import pytest
from tests.test_onnx.utils import run_node
from tests.tests_python.utils import run_node
def import_and_compute(op_type, input_data, **node_attrs):
src/bindings/python/tests/test_onnx/test_ops_random.py → src/frontends/onnx/tests/tests_python/test_ops_random.py
+1 -1
View File
@@ -6,7 +6,7 @@ import numpy as np
import onnx
import onnx.mapping
from tests.test_onnx.utils import run_node
from tests.tests_python.utils import run_node
def test_random_uniform():
src/bindings/python/tests/test_onnx/test_ops_reduction.py → src/frontends/onnx/tests/tests_python/test_ops_reduction.py
+1 -1
View File
@@ -8,7 +8,7 @@ import pytest
from tests import xfail_issue_99962
from tests.runtime import get_runtime
from tests.test_onnx.utils import (
from tests.tests_python.utils import (
run_node,
import_onnx_model,
)
src/bindings/python/tests/test_onnx/test_ops_reshape.py → src/frontends/onnx/tests/tests_python/test_ops_reshape.py
+1 -1
View File
@@ -8,7 +8,7 @@ import pytest
from onnx.helper import make_graph, make_model, make_node, make_tensor_value_info
from tests.runtime import get_runtime
from tests.test_onnx.utils import (
from tests.tests_python.utils import (
all_arrays_equal,
get_node_model,
import_onnx_model,
src/bindings/python/tests/test_onnx/test_ops_unary.py → src/frontends/onnx/tests/tests_python/test_ops_unary.py
+1 -1
View File
@@ -10,7 +10,7 @@ from onnx.helper import make_graph, make_model, make_node, make_tensor_value_inf
from openvino.runtime.exceptions import OVTypeError
from tests.runtime import get_runtime
from tests.test_onnx.utils import get_node_model, import_onnx_model, run_model, run_node
from tests.tests_python.utils import get_node_model, import_onnx_model, run_model, run_node
@pytest.mark.parametrize(
src/bindings/python/tests/test_onnx/test_ops_variadic.py → src/frontends/onnx/tests/tests_python/test_ops_variadic.py
+1 -1
View File
@@ -8,7 +8,7 @@ import numpy as np
import onnx
import pytest
from tests.test_onnx.utils import run_node
from tests.tests_python.utils import run_node
@pytest.mark.parametrize(
src/bindings/python/tests/test_onnx/test_zoo_models.py → src/frontends/onnx/tests/tests_python/test_zoo_models.py
+2 -2
View File
@@ -7,8 +7,8 @@ from pathlib import Path
from typing import Sequence, Any
import numpy as np
from tests.test_onnx.utils import OpenVinoOnnxBackend
from tests.test_onnx.utils.model_importer import ModelImportRunner
from tests.tests_python.utils import OpenVinoOnnxBackend
from tests.tests_python.utils.model_importer import ModelImportRunner
from tests import (
xfail_issue_67415,
src/bindings/python/tests/test_onnx/utils/__init__.py → src/frontends/onnx/tests/tests_python/utils/__init__.py
+2 -2
View File
@@ -11,8 +11,8 @@ from onnx.helper import make_graph, make_model, make_node, make_tensor_value_inf
import tests
from tests.runtime import get_runtime
from tests.test_onnx.utils.onnx_backend import OpenVinoOnnxBackend
from tests.test_onnx.utils.onnx_helpers import import_onnx_model
from tests.tests_python.utils.onnx_backend import OpenVinoOnnxBackend
from tests.tests_python.utils.onnx_helpers import import_onnx_model
def run_node(onnx_node, data_inputs, **kwargs):
src/bindings/python/tests/test_onnx/utils/model_importer.py → src/frontends/onnx/tests/tests_python/utils/model_importer.py
+1 -1
View File
@@ -14,7 +14,7 @@ from onnx.backend.base import Backend, BackendRep
from onnx.backend.test.case.test_case import TestCase as OnnxTestCase
from onnx.backend.test.runner import TestItem
from pathlib import Path
from tests.test_onnx.utils.onnx_helpers import import_onnx_model
from tests.tests_python.utils.onnx_helpers import import_onnx_model
from typing import (
Any,
Dict,
src/bindings/python/tests/test_onnx/utils/onnx_backend.py → src/frontends/onnx/tests/tests_python/utils/onnx_backend.py
+1 -1
View File
@@ -17,7 +17,7 @@ from onnx.helper import make_graph, make_model, make_tensor_value_info, np_dtype
from openvino.runtime import Model
from tests.runtime import get_runtime
from tests.test_onnx.utils.onnx_helpers import import_onnx_model
from tests.tests_python.utils.onnx_helpers import import_onnx_model
class OpenVinoOnnxBackendRep(BackendRep):
src/bindings/python/tests/test_onnx/utils/onnx_helpers.py → src/frontends/onnx/tests/tests_python/utils/onnx_helpers.py
View File
File renamed without changes.