diff --git a/.ci/azure/linux.yml b/.ci/azure/linux.yml index 8a3f9cb563d..3e679e565d1 100644 --- a/.ci/azure/linux.yml +++ b/.ci/azure/linux.yml @@ -270,6 +270,11 @@ jobs: displayName: 'OV Core UT' continueOnError: false + - script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/onnx_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:TEST-ONNXImportUT.xml + workingDirectory: $(INSTALL_TEST_DIR) + displayName: 'ONNX Frontend UT' + continueOnError: false + - script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/paddle_tests --gtest_print_time=1 --gtest_output=xml:TEST-Paddle.xml displayName: 'Paddle Frontend UT' continueOnError: false diff --git a/.ci/azure/mac.yml b/.ci/azure/mac.yml index f73cea49d9e..350312adb06 100644 --- a/.ci/azure/mac.yml +++ b/.ci/azure/mac.yml @@ -152,12 +152,17 @@ jobs: - script: ls -alR $(INSTALL_DIR) displayName: 'List install files' - - script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU*:IE_CPU.onnx_model_sigmoid:IE_CPU/GRUSequenceOp.onnx_model_gru* --gtest_output=xml:TEST-NGraphUT.xml + - script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-NGraphUT.xml workingDirectory: $(INSTALL_TEST_DIR) displayName: 'OV Core UT' continueOnError: false enabled: false + - script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/onnx_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU*:IE_CPU.onnx_model_sigmoid:IE_CPU/GRUSequenceOp.onnx_model_gru* --gtest_output=xml:TEST-ONNXImportUT.xml + workingDirectory: $(INSTALL_TEST_DIR) + displayName: 'ONNX Frontend UT' + continueOnError: false + - script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/InferenceEngineUnitTests --gtest_print_time=1 --gtest_filter=-MKLDNNGraphStructureTests.TestNoRedundantReordersBeforeDWConvolution:TestConvolution/MKLDNNGraphConvolutionTests.TestsConvolution/0:TestConvolutionDefaultPrimitivesPriority/MKLDNNGraphConvolutionTests.TestsConvolution/0 --gtest_output=xml:TEST-InferenceEngineUnitTests.xml displayName: 'IE UT old' continueOnError: false diff --git a/.ci/azure/windows.yml b/.ci/azure/windows.yml index e78e8c9cc6a..a761c838786 100644 --- a/.ci/azure/windows.yml +++ b/.ci/azure/windows.yml @@ -211,6 +211,11 @@ jobs: displayName: 'OV Core UT' continueOnError: false + - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\onnx_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:TEST-ONNXImportUT.xml + workingDirectory: $(INSTALL_TEST_DIR) + displayName: 'ONNX Frontend UT' + continueOnError: false + - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\paddle_tests --gtest_print_time=1 --gtest_output=xml:TEST-Paddle.xml displayName: 'Paddle Frontend UT' continueOnError: false diff --git a/src/core/tests/CMakeLists.txt b/src/core/tests/CMakeLists.txt index 42669f646a1..3143a91d41a 100644 --- a/src/core/tests/CMakeLists.txt +++ b/src/core/tests/CMakeLists.txt @@ -11,14 +11,6 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT ENABLE_SANITIZER) endif() add_definitions(-DSERIALIZED_ZOO=\"${TEST_MODEL_ZOO}/core/models\") -set(OV_ONNX_NAMESPACE ngraph_onnx) - -if(ENABLE_TESTS) - add_definitions("-DTEST_FILES=\"${TEST_MODEL_ZOO}/core/files\"") - if (ENABLE_OV_ONNX_FRONTEND) - add_subdirectory(onnx_test_util) - endif() -endif() if(NOT ENABLE_OV_CORE_UNIT_TESTS) message(STATUS "nGraph unit tests disabled") @@ -503,30 +495,6 @@ set(OP_EVAL_TEST_SRC op_eval/transpose.cpp op_eval/variadic_split.cpp) -if (ENABLE_OV_ONNX_FRONTEND) - list(APPEND MULTI_TEST_SRC - onnx/onnx_import.in.cpp - onnx/onnx_import_com_microsoft.in.cpp - onnx/onnx_import_controlflow.in.cpp - onnx/onnx_import_const_folding.in.cpp - onnx/onnx_import_convpool.in.cpp - onnx/onnx_import_deprecated.in.cpp - onnx/onnx_import_dyn_shapes.in.cpp - onnx/onnx_import_external_data.in.cpp - onnx/onnx_import_org_openvino.in.cpp - onnx/onnx_import_reshape.in.cpp - onnx/onnx_import_rnn.in.cpp - onnx/onnx_import_quant.in.cpp - onnx/onnx_test_utils.in.cpp - onnx/onnx_import_with_editor.in.cpp) - list(APPEND SRC - onnx/onnx_import_exceptions.cpp - onnx/onnx_import_library.cpp - onnx/onnx_tensor_names.cpp - onnx/onnx_transformations.cpp - onnx/onnx_editor.cpp) -endif() - # SOURCE FOR FRONTEND TESTING file(GLOB FRONTEND_TESTS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/frontend/frontend_manager.cpp ${CMAKE_CURRENT_SOURCE_DIR}/frontend/decoder_transformation_extension.cpp @@ -596,18 +564,7 @@ target_link_libraries(ov_core_unit_tests PRIVATE ngraph_test_util openvino::conditional_compilation openvino::runtime::dev) -# Protobuf-lite does not support parsing files from prototxt format -# Since most of the onnx models are stored in this format it have to be disabled if (ENABLE_OV_ONNX_FRONTEND) - # It's needed by onnx_import_library.cpp and onnx_import_exceptions.cpp tests to include onnx_pb.h. - # Not linking statically to libprotobuf (linked into libonnx) avoids false-failing onnx_editor tests. - target_include_directories(ov_core_unit_tests - SYSTEM PRIVATE - $ - $ - ${Protobuf_INCLUDE_DIRS}) - target_compile_definitions(ov_core_unit_tests - PRIVATE $) target_compile_definitions(ov_core_unit_tests PRIVATE ENABLE_OV_ONNX_FRONTEND) endif() @@ -615,17 +572,6 @@ if (OV_COMPILER_IS_CLANG) target_compile_options(ov_core_unit_tests PRIVATE -Wno-undef -Wno-reserved-id-macro) endif() -if (ENABLE_OV_ONNX_FRONTEND) - get_target_property(ONNX_FRONTEND_SRC_DIR openvino_onnx_frontend SOURCE_DIR) - target_include_directories(ov_core_unit_tests PRIVATE ${ONNX_FRONTEND_SRC_DIR}/src) - target_link_libraries(ov_core_unit_tests PRIVATE openvino_onnx_frontend onnx_test_util) - if (LINUX) - target_link_options(ov_core_unit_tests PRIVATE -Wl,--exclude-libs,ALL) - elseif(APPLE) - target_link_options(ov_core_unit_tests PRIVATE -Wl,-dead_strip) - endif() -endif() - if(ENABLE_OV_IR_FRONTEND) add_dependencies(ov_core_unit_tests openvino_ir_frontend) endif() diff --git a/src/frontends/tests/CMakeLists.txt b/src/frontends/tests/CMakeLists.txt index 42d7b63bb56..3500cc9bcf6 100644 --- a/src/frontends/tests/CMakeLists.txt +++ b/src/frontends/tests/CMakeLists.txt @@ -5,3 +5,7 @@ set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE OFF) add_subdirectory(frontend/shared) + +if (ENABLE_OV_ONNX_FRONTEND) + add_subdirectory(onnx) +endif() diff --git a/src/frontends/tests/onnx/CMakeLists.txt b/src/frontends/tests/onnx/CMakeLists.txt new file mode 100644 index 00000000000..074c1865241 --- /dev/null +++ b/src/frontends/tests/onnx/CMakeLists.txt @@ -0,0 +1,156 @@ +# Copyright (C) 2018-2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE OFF) + +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT ENABLE_SANITIZER) + # gold linker on ubuntu20.04 may fail to link binaries build with sanitizer + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fuse-ld=gold") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fuse-ld=gold") +endif() + +message(STATUS "ONNX frontend tests enabled") + +add_compile_definitions( + SERIALIZED_ZOO="${TEST_MODEL_ZOO}/core/models" + TEST_FILES="${TEST_MODEL_ZOO}/core/files") + +add_subdirectory(test_util) + +if(LINUX) + set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) +endif() + +list(APPEND ONNX_TESTS_DEPENDENCIES openvino_template_extension) + +if (ENABLE_INTEL_CPU AND ENABLE_OV_CORE_BACKEND_UNIT_TESTS) + message(STATUS "OV_CORE_TESTS: IE:CPU enabled") + set(ACTIVE_BACKEND_LIST ${ACTIVE_BACKEND_LIST} "IE:CPU") + if (ENABLE_STRICT_DEPENDENCIES) + # For convinience add a runtime dependency to build along with this target. + # Warning: Parallel build with -GNinja may not be efficient. + list(APPEND ONNX_TESTS_DEPENDENCIES openvino_intel_cpu_plugin) + endif() +endif() + +if (ENABLE_INTEL_GPU AND ENABLE_OV_CORE_BACKEND_UNIT_TESTS) + message(STATUS "OV_CORE_TESTS: IE:GPU enabled") + set(ACTIVE_BACKEND_LIST ${ACTIVE_BACKEND_LIST} "IE:GPU") + if (ENABLE_STRICT_DEPENDENCIES) + # For convinience add a runtime dependency to build along with this target. + # Warning: Parallel build with -GNinja may not be efficient. + list(APPEND ONNX_TESTS_DEPENDENCIES openvino_intel_gpu_plugin) + endif() +endif() + +if (ENABLE_TEMPLATE AND ENABLE_OV_CORE_BACKEND_UNIT_TESTS) + message(STATUS "OV_CORE_TESTS: INTERPRETER enabled") + set(ACTIVE_BACKEND_LIST ${ACTIVE_BACKEND_LIST} INTERPRETER) + if (ENABLE_STRICT_DEPENDENCIES) + list(APPEND ONNX_TESTS_DEPENDENCIES openvino_template_plugin) + endif() +endif() + +# backend specific test files must meet the following requirements: +# 1) The must be named .in.cpp +# 2) They must be in the `frontends/tests/onnx` directory +# 3) Include "util/test_control.hpp" in your cpp file +# 4) add the line `static string s_manifest = "${MANIFEST}";` to your cpp file +# 5) Use the `NGRAPH_TEST` macro in place of `TEST`. +# All such files are configured via cmake which replaces all instances of cmake variables +# such as ${BACKEND_NAME} with their values, such as CPU, GPU, or INTERPRETER. + +set(MULTI_TEST_SRC + onnx_import.in.cpp + onnx_import_com_microsoft.in.cpp + onnx_import_controlflow.in.cpp + onnx_import_const_folding.in.cpp + onnx_import_convpool.in.cpp + onnx_import_deprecated.in.cpp + onnx_import_dyn_shapes.in.cpp + onnx_import_external_data.in.cpp + onnx_import_org_openvino.in.cpp + onnx_import_reshape.in.cpp + onnx_import_rnn.in.cpp + onnx_import_quant.in.cpp + onnx_test_utils.in.cpp + onnx_import_with_editor.in.cpp) +set(SRC + onnx_import_exceptions.cpp + onnx_import_library.cpp + onnx_tensor_names.cpp + onnx_transformations.cpp + onnx_editor.cpp) + +foreach(src IN LISTS SRC MULTI_TEST_SRC) + if(IS_ABSOLUTE "${src}") + list(APPEND full_src_names ${src}) + else() + list(APPEND full_src_names "${CMAKE_CURRENT_SOURCE_DIR}/${src}") + endif() +endforeach() +add_clang_format_target(onnx_tests_clang FOR_SOURCES ${full_src_names}) + +foreach(BACKEND_NAME ${ACTIVE_BACKEND_LIST}) + string(TOLOWER ${BACKEND_NAME} BACKEND_DIR) + string(REGEX REPLACE "([a-z0-9]+):(.*)" "\\1" BACKEND_DIR ${BACKEND_DIR}) + set(MANIFEST ${CMAKE_CURRENT_SOURCE_DIR}/runtime/${BACKEND_DIR}/unit_test.manifest) + + foreach(TEST_SRC ${MULTI_TEST_SRC}) + string(REPLACE ":" "_" BACKEND_NAME ${BACKEND_NAME}) + string(REPLACE ".in." "_${BACKEND_NAME}." TARGET_NAME ${TEST_SRC}) + configure_file(${TEST_SRC} ${TARGET_NAME}) + set(SRC ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} ${SRC}) + endforeach() + + message(STATUS "Adding unit test for backend ${BACKEND_NAME}") +endforeach() + +add_executable(onnx_tests ${SRC}) + +add_dependencies(onnx_tests template_extension) + +target_include_directories(onnx_tests PRIVATE ".") + +target_compile_definitions(onnx_tests + PRIVATE + SHARED_LIB_PREFIX="${CMAKE_SHARED_LIBRARY_PREFIX}" + SHARED_LIB_SUFFIX="${IE_BUILD_POSTFIX}${CMAKE_SHARED_LIBRARY_SUFFIX}") + +if(ONNX_TESTS_DEPENDENCIES) + add_dependencies(onnx_tests ${ONNX_TESTS_DEPENDENCIES}) +endif() + +target_link_libraries(onnx_tests PRIVATE commonTestUtils engines_test_util onnx_test_util openvino::runtime::dev + openvino_onnx_frontend) + +# It's needed by onnx_import_library.cpp and onnx_import_exceptions.cpp tests to include onnx_pb.h. +# Not linking statically to libprotobuf (linked into libonnx) avoids false-failing onnx_editor tests. +target_include_directories(onnx_tests + SYSTEM PRIVATE + $ + $ + ${Protobuf_INCLUDE_DIRS}) +target_compile_definitions(onnx_tests PRIVATE $) +target_compile_definitions(onnx_tests PRIVATE ENABLE_OV_ONNX_FRONTEND) + +if (OV_COMPILER_IS_CLANG) + target_compile_options(onnx_tests PRIVATE -Wno-undef -Wno-reserved-id-macro) +endif() + +target_include_directories(onnx_tests PRIVATE + $) +if (LINUX) + target_link_options(onnx_tests PRIVATE -Wl,--exclude-libs,ALL) +elseif(APPLE) + target_link_options(onnx_tests PRIVATE -Wl,-dead_strip) +endif() + +install(TARGETS onnx_tests + RUNTIME DESTINATION tests + COMPONENT tests + EXCLUDE_FROM_ALL) + +# process models +add_dependencies(onnx_tests test_model_zoo) diff --git a/src/core/tests/onnx/onnx_editor.cpp b/src/frontends/tests/onnx/onnx_editor.cpp similarity index 100% rename from src/core/tests/onnx/onnx_editor.cpp rename to src/frontends/tests/onnx/onnx_editor.cpp diff --git a/src/core/tests/onnx/onnx_import.in.cpp b/src/frontends/tests/onnx/onnx_import.in.cpp similarity index 100% rename from src/core/tests/onnx/onnx_import.in.cpp rename to src/frontends/tests/onnx/onnx_import.in.cpp diff --git a/src/core/tests/onnx/onnx_import_com_microsoft.in.cpp b/src/frontends/tests/onnx/onnx_import_com_microsoft.in.cpp similarity index 100% rename from src/core/tests/onnx/onnx_import_com_microsoft.in.cpp rename to src/frontends/tests/onnx/onnx_import_com_microsoft.in.cpp diff --git a/src/core/tests/onnx/onnx_import_const_folding.in.cpp b/src/frontends/tests/onnx/onnx_import_const_folding.in.cpp similarity index 100% rename from src/core/tests/onnx/onnx_import_const_folding.in.cpp rename to src/frontends/tests/onnx/onnx_import_const_folding.in.cpp diff --git a/src/core/tests/onnx/onnx_import_controlflow.in.cpp b/src/frontends/tests/onnx/onnx_import_controlflow.in.cpp similarity index 100% rename from src/core/tests/onnx/onnx_import_controlflow.in.cpp rename to src/frontends/tests/onnx/onnx_import_controlflow.in.cpp diff --git a/src/core/tests/onnx/onnx_import_convpool.in.cpp b/src/frontends/tests/onnx/onnx_import_convpool.in.cpp similarity index 100% rename from src/core/tests/onnx/onnx_import_convpool.in.cpp rename to src/frontends/tests/onnx/onnx_import_convpool.in.cpp diff --git a/src/core/tests/onnx/onnx_import_deprecated.in.cpp b/src/frontends/tests/onnx/onnx_import_deprecated.in.cpp similarity index 100% rename from src/core/tests/onnx/onnx_import_deprecated.in.cpp rename to src/frontends/tests/onnx/onnx_import_deprecated.in.cpp diff --git a/src/core/tests/onnx/onnx_import_dyn_shapes.in.cpp b/src/frontends/tests/onnx/onnx_import_dyn_shapes.in.cpp similarity index 100% rename from src/core/tests/onnx/onnx_import_dyn_shapes.in.cpp rename to src/frontends/tests/onnx/onnx_import_dyn_shapes.in.cpp diff --git a/src/core/tests/onnx/onnx_import_exceptions.cpp b/src/frontends/tests/onnx/onnx_import_exceptions.cpp similarity index 100% rename from src/core/tests/onnx/onnx_import_exceptions.cpp rename to src/frontends/tests/onnx/onnx_import_exceptions.cpp diff --git a/src/core/tests/onnx/onnx_import_external_data.in.cpp b/src/frontends/tests/onnx/onnx_import_external_data.in.cpp similarity index 100% rename from src/core/tests/onnx/onnx_import_external_data.in.cpp rename to src/frontends/tests/onnx/onnx_import_external_data.in.cpp diff --git a/src/core/tests/onnx/onnx_import_library.cpp b/src/frontends/tests/onnx/onnx_import_library.cpp similarity index 100% rename from src/core/tests/onnx/onnx_import_library.cpp rename to src/frontends/tests/onnx/onnx_import_library.cpp diff --git a/src/core/tests/onnx/onnx_import_org_openvino.in.cpp b/src/frontends/tests/onnx/onnx_import_org_openvino.in.cpp similarity index 100% rename from src/core/tests/onnx/onnx_import_org_openvino.in.cpp rename to src/frontends/tests/onnx/onnx_import_org_openvino.in.cpp diff --git a/src/core/tests/onnx/onnx_import_quant.in.cpp b/src/frontends/tests/onnx/onnx_import_quant.in.cpp similarity index 100% rename from src/core/tests/onnx/onnx_import_quant.in.cpp rename to src/frontends/tests/onnx/onnx_import_quant.in.cpp diff --git a/src/core/tests/onnx/onnx_import_reshape.in.cpp b/src/frontends/tests/onnx/onnx_import_reshape.in.cpp similarity index 100% rename from src/core/tests/onnx/onnx_import_reshape.in.cpp rename to src/frontends/tests/onnx/onnx_import_reshape.in.cpp diff --git a/src/core/tests/onnx/onnx_import_rnn.in.cpp b/src/frontends/tests/onnx/onnx_import_rnn.in.cpp similarity index 100% rename from src/core/tests/onnx/onnx_import_rnn.in.cpp rename to src/frontends/tests/onnx/onnx_import_rnn.in.cpp diff --git a/src/core/tests/onnx/onnx_import_with_editor.in.cpp b/src/frontends/tests/onnx/onnx_import_with_editor.in.cpp similarity index 100% rename from src/core/tests/onnx/onnx_import_with_editor.in.cpp rename to src/frontends/tests/onnx/onnx_import_with_editor.in.cpp diff --git a/src/core/tests/onnx/onnx_tensor_names.cpp b/src/frontends/tests/onnx/onnx_tensor_names.cpp similarity index 100% rename from src/core/tests/onnx/onnx_tensor_names.cpp rename to src/frontends/tests/onnx/onnx_tensor_names.cpp diff --git a/src/core/tests/onnx/onnx_test_utils.in.cpp b/src/frontends/tests/onnx/onnx_test_utils.in.cpp similarity index 100% rename from src/core/tests/onnx/onnx_test_utils.in.cpp rename to src/frontends/tests/onnx/onnx_test_utils.in.cpp diff --git a/src/core/tests/onnx/onnx_transformations.cpp b/src/frontends/tests/onnx/onnx_transformations.cpp similarity index 100% rename from src/core/tests/onnx/onnx_transformations.cpp rename to src/frontends/tests/onnx/onnx_transformations.cpp diff --git a/src/frontends/tests/onnx/runtime/ie/unit_test.manifest b/src/frontends/tests/onnx/runtime/ie/unit_test.manifest new file mode 100644 index 00000000000..7f8bfa6d465 --- /dev/null +++ b/src/frontends/tests/onnx/runtime/ie/unit_test.manifest @@ -0,0 +1,424 @@ +#------------------------------------------------------------------------------- +# +# Inference Engine all plugins excludes +# +#------------------------------------------------------------------------------- + +#------------------------------------------------------------------------------- +# +# nGraph ONNX C++ Importer unit tests +# +#------------------------------------------------------------------------------- + +# Quantize layer input 'Multiply_7' doesn't have blobs +onnx_model_quantize_linear +onnx_model_quantize_linear_zero_point +onnx_model_quantize_linear_axis_zero +onnx_model_quantize_linear_axis_negative +onnx_expand_function + +# DequantizeLinear: +# C++ exception with description "Unsupported precisions! +IE_CPU.onnx_model_dequantize_linear_scalar_zero_scale_int8 +IE_CPU.onnx_model_dequantize_linear_1d_zero_scale_int8 +# C++ exception with description "Input data precision not supported. Expected float. +IE_CPU.onnx_model_dequantize_linear_1d_zero_scale_int8_4d + + +# Result mismatch +onnx_model_shape +onnx_model_split_equal_parts_default +onnx_model_argmin_no_keepdims +onnx_model_elu +onnx_model_top_k +onnx_model_erf +onnx_model_addmul_abc + +# data [] doesn't exist +onnx_model_tile +onnx_model_tile_static +onnx_model_logsoftmax_0D + +# nGraph function's output number 0 was not found in the CNNNetwork built from it. +onnx_model_split_equal_parts_2d +onnx_model_split_variable_parts_2d +onnx_top_k_opset_10_const_k +onnx_top_k_opset_11_const_k_smallest +onnx_top_k_opset_11_const_k_smallest_negative_axis + +# [NOT_IMPLEMENTED] Input image format BOOL is not supported yet... +onnx_constant_sparse_tensor_boolean_3x4 + +# Layer input port 1 is not connected to any data +onnx_model_conv_transpose_w_groups + +# [NOT_IMPLEMENTED] Input image format I64 is not supported yet... +onnx_model_global_lp_pool_p0 +onnx_model_mod_sign_i64 +onnx_model_constant_of_shape_float_zeros +onnx_model_constant_of_shape_int_ones +onnx_dyn_shapes_slice_10_3d_input_12_axes +IE_CPU.onnx_dyn_shapes_slice_10_default_axes +onnx_dyn_shapes_ab_plus_c_inference +onnx_dyn_shapes_dynamic_rank_input_inference +onnx_dyn_shapes_model_conv_with_dynamic_batch +onnx_dyn_shapes_model_conv_with_dynamic_bias +onnx_dyn_shapes_expand_1_dyn_shape +onnx_dyn_shapes_expand_2_dyn_shape +onnx_dyn_shapes_expand_3_dyn_shape +onnx_dyn_shapes_expand_4_dyn_shape +onnx_dyn_shapes_expand_5_dyn_shape +onnx_dyn_shapes_expand_6_dyn_shape +onnx_dyn_shapes_expand_uint16_dyn_shape +onnx_dyn_shapes_model_global_lp_dynamic_hw +onnx_dyn_shapes_slice_10_2d_input +onnx_dyn_shapes_slice_10_default_steps +onnx_dyn_shapes_slice_10_slice_2d_default_steps_dyn_begin_end +onnx_dyn_shapes_slice_10_clamp_neg_ends +onnx_dyn_shapes_slice_10_3d_input +onnx_dyn_shapes_slice_10_3d_input_neg_axes +onnx_dyn_shapes_slice_10_3d_input_20_axes +onnx_dyn_shapes_slice_10_4d_input_23_axes +onnx_dyn_shapes_slice_10_4d_input_0231_axes_ends_max +onnx_dyn_shapes_slice_10_4d_input_2103_axes_ends_max +onnx_dyn_shapes_slice_10_4d_input_23_axes_21_steps +onnx_dyn_shapes_slice_10_3d_input_12_axes +onnx_top_k_opset_10 +onnx_model_scatterND_param_i64_indices +IE_CPU.onnx_constant_sparse_tensor_int64_3x4 +IE_CPU.onnx_constant_sparse_tensor_uint64_3x4 + +# I64 ONNX Clip Failing on ubuntu18/20_release CI +IE_CPU.onnx_clip_no_min_no_max_int64 # -2147483648 is not close to 2147483647 at index 2 +IE_CPU.onnx_clip_no_min_set_max_int64 # -1 is not close to 2147483647 at index 0 + +# Constant has zero dimension that is not allowable +onnx_dyn_shapes_transpose + +# y Incorrect input precision. Only FP32 is supported! +onnx_model_erf_int32 + +# Could not eliminate all Dyn nodes +onnx_model_conv_with_dynamic_batch + +# Segmentation fault +onnx_model_fake_quantize_nonconst_inputs_infer +onnx_model_quantize_linear_const_scale_const_zero_p + +# OneHot operation has a form that is not supported +onnx_model_one_hot_without_axis +onnx_model_one_hot_with_axis + +# Dynamic function 'get_shape was called on a descriptor::Tensor with dynamic shape' +onnx_dyn_shapes_model_acosh_1_3 +onnx_dyn_shapes_model_acosh_3_2 +onnx_dyn_shapes_model_asinh_1_3 +onnx_dyn_shapes_model_asinh_3_2 +onnx_dyn_shapes_model_atanh_1_3 +onnx_dyn_shapes_model_atanh_3_2 +onnx_dyn_shapes_avg_pool_dyn_shape +onnx_dyn_shapes_max_pool_dyn_shape +onnx_dyn_shapes_global_avg_pool_dyn_shape +onnx_dyn_shapes_global_max_pool_dyn_shape +onnx_dyn_shapes_model_flatten +onnx_model_instance_normalization_dyn_shape +onnx_model_instance_normalization_dyn_shape2 +onnx_dyn_shapes_arg_max_dyn_shape +onnx_dyn_shapes_arg_min_no_keep_dims_dyn_shape +onnx_dyn_shapes_model_tile +onnx_dyn_shapes_model_convtranspose_dyn_data +onnx_dyn_shapes_model_convtranspose_dyn_filters +onnx_dyn_shapes_flatten_axis_0 +onnx_dyn_shapes_flatten_axis +onnx_dyn_shapes_flatten_neg_axis +onnx_model_range_positive_step +onnx_model_range_negative_step +onnx_dyn_shapes_slice_1_3d_input_21_axes_ends_max +onnx_model_max_pool_dyn_rank_without_default_attrs +onnx_model_lp_norm_default_dynamic +onnx_instance_normalization_dynamic + +# (Constant W, R inputs are required) Ticket: 49207 +# Function references undeclared parameters +# LSTMSequence +IE_CPU.onnx_model_lstm_fwd_large_batch_no_clip +IE_CPU.onnx_model_lstm_fwd_mixed_seq +IE_CPU.onnx_model_lstm_mixed_seq_reverse +# GRUSequence +IE_CPU.onnx_model_gru_defaults_fwd +IE_CPU.onnx_model_gru_fwd_mixed_seq_len +IE_CPU.onnx_model_gru_rev_clip +IE_CPU.onnx_model_gru_reverse +IE_CPU.onnx_model_gru_fwd_bias_initial_h +IE_CPU.onnx_model_gru_bidirectional +IE_CPU.onnx_model_gru_fwd_linear_before_reset +# RNNSequence +IE_CPU.onnx_model_rnn_defaults_fwd +IE_CPU.onnx_model_rnn_fwd_activations +IE_CPU.onnx_model_rnn_fwd_mixed_seq_len +IE_CPU.onnx_model_rnn_rev_clip +IE_CPU.onnx_model_rnn_reverse +IE_CPU.onnx_model_rnn_fwd_bias_initial_h +IE_CPU.onnx_model_rnn_bidirectional + +# Legacy tests with unsupported features from opset4 LSTM/GRU/RNN +# Peepholes input unsupported +onnx_model_lstm_fwd_with_clip_peepholes +onnx_model_lstm_bdir_short_input_seq_peepholes +# Activation function hardsigmoid is not supported +onnx_model_gru_fwd_activations_relu_hardsigmoid +onnx_model_lstm_fwd_hardsigmoid_activation + +# Unsupported dynamic ops +onnx_size_dyn_op +onnx_model_gru_defaults_fwd_const_dynamic +onnx_model_rnn_defaults_fwd_const_dynamic +onnx_model_depth_to_space_dynamic_input +onnx_model_space_to_depth_dynamic_input +onnx_model_eye_like_dyn_shape +onnx_model_eye_like_dyn_rank + + +# Constant network +# intel_cpu::Graph::CreateGraph: No inputs for the topology +onnx_size_op_single +onnx_size_op_graph_end +onnx_size_op_graph_middle + +# /openvino/src/plugins/intel_cpu/graph.cpp:747 +# Output blob byte size is not equal network output byte size (64!=216)." thrown in the test body. +onnx_model_quant_conv_linear_3d + + +# Result mismatch +onnx_dyn_shapes_model_tile_static + +# [NOT_IMPLEMENTED] Output format I64 is not supported yet... +onnx_constant_integer_scalar +onnx_constant_integer_array +onnx_dyn_shapes_max_pool_with_indices_output +onnx_model_max_pool_3d +onnx_model_max_pool_4d_ceil_mode +onnx_model_max_pool_4d_dilations +onnx_model_max_pool_4d_strides +onnx_model_max_pool_4d_ceil_strides + +# Unsupported primitive of type: ROIAlign +IE_CPU.onnx_roi_align_f32 + +#------------------------------------------------------------------------------- +# +# Inference Engine CPU plugin excludes +# +#------------------------------------------------------------------------------- + +# Segfault +IE_CPU.onnx_model_reverse_sequence_1_batch_0 +IE_CPU.onnx_model_reverse_sequence_0_batch_1 + +# nGraph Interpolate operation with name: Y cannot be converted to Interpolate layer with name: +# Y because output with index 0 contains dynamic shapes: {?,?,?,?} +IE_CPU.onnx_resize11_scales_nearest_asymmetric_floor_dynamic_sizes +IE_CPU.onnx_resize11_up_sizes_cubic_half_pixel_dynamic_sizes + +# Unsupported dynamic ops: v3::NonMaxSuppressionIE3 +IE_CPU.onnx_model_nonmaxsuppression_center_point_box_format +IE_CPU.onnx_model_nonmaxsuppression_single_box + +# Unsupported dynamic op +IE_CPU.onnx_model_reduce_sum_13_axes_as_input +IE_CPU.onnx_model_reduce_sum_13_input_dynamic +IE_CPU.onnx_model_reduce_sum_13_axes_empty_dynamic_rank_input +IE_CPU.onnx_model_reduce_sum_dynamic_rank_input + +# Axes has zero dimension which is not allowed +IE_CPU.onnx_model_reduce_sum_13_axes_as_0_dim_input + +# nGraph test infrastructure doesn't support (b)float16 input data in IE_CPU-tests +IE_CPU.onnx_constant_sparse_tensor_float16_3x4 +IE_CPU.onnx_float16_tensor_as_int32 + +# The plugin does not support UNSPECIFIED +IE_CPU.onnx_constant_sparse_tensor_uint32_3x4 +IE_CPU.onnx_constant_sparse_tensor_double_3x4 + +# Sparse tensor with (u)i16 type tests +IE_CPU.onnx_constant_sparse_tensor_int16_3x4 +IE_CPU.onnx_constant_sparse_tensor_uint16_3x4 + +#------------------------------------------------------------------------------- +# +# Inference Engine GPU plugin excludes +# +#------------------------------------------------------------------------------- + +IE_GPU.onnx_model_reshape_reduced_dims +IE_GPU.onnx_model_reshape_reordered_dims +IE_GPU.onnx_model_reshape_extended_dims +IE_GPU.onnx_model_reshape_single_dim +IE_GPU.onnx_model_reshape_negative_dim +IE_GPU.onnx_model_reshape_negative_with_zero_dim +IE_GPU.onnx_model_reshape_output_shape_as_input +IE_GPU.onnx_model_depth_to_space +IE_GPU.onnx_model_space_to_depth +IE_GPU.onnx_model_squeeze +IE_GPU.onnx_model_unsqueeze +IE_GPU.onnx_model_unsqueeze_negative_axes +IE_GPU.onnx_model_concat +IE_GPU.onnx_model_concat_negative_axis +IE_GPU.onnx_model_flatten +IE_GPU.onnx_model_average_pool_2d +IE_GPU.onnx_model_average_pool_2d_pads +IE_GPU.onnx_model_max_pool_2d_pads +IE_GPU.onnx_model_global_lp_pool_p1 +IE_GPU.onnx_test_test_case +IE_GPU.onnx_test_test_case_mutliple_inputs +IE_GPU.onnx_model_add_abc +IE_GPU.onnx_model_binary_add_abc +IE_GPU.onnx_model_add_abc_initializers +IE_GPU.onnx_model_override_op +IE_GPU.onnx_model_custom_op +IE_GPU.onnx_model_custom_op_default_domain +IE_GPU.onnx_model_missing_op_domain +IE_GPU.onnx_model_missing_input +IE_GPU.onnx_model_initializer_wo_input +IE_GPU.onnx_model_addmul_abc +IE_GPU.onnx_model_relu +IE_GPU.onnx_model_sum_opset1 +IE_GPU.onnx_model_sum +IE_GPU.onnx_model_min_two_inputs_opset1 +IE_GPU.onnx_model_min_two_inputs +IE_GPU.onnx_model_max_opset1 +IE_GPU.onnx_model_max +IE_GPU.onnx_model_gemm_abc +IE_GPU.onnx_model_matmul +IE_GPU.onnx_model_sub +IE_GPU.onnx_model_div +IE_GPU.onnx_model_add_bcast +IE_GPU.onnx_model_reduce_log_sum +IE_GPU.onnx_model_reduce_log_sum_exp +IE_GPU.onnx_model_reduce_l1 +IE_GPU.onnx_model_reduce_max +IE_GPU.onnx_model_reduce_mean +IE_GPU.onnx_model_reduce_min +IE_GPU.onnx_model_reduce_prod +IE_GPU.onnx_model_reduce_sum +IE_GPU.onnx_model_reduce_sum_square +IE_GPU.onnx_model_leaky_relu +IE_GPU.onnx_model_selu +IE_GPU.onnx_model_sigmoid +IE_GPU.onnx_model_tanh +IE_GPU.onnx_model_thresholded_relu +IE_GPU.onnx_model_matmul_vec_ten3d +IE_GPU.onnx_model_softplus +IE_GPU.onnx_model_softplus_infinity +IE_GPU.onnx_model_sum_opset8 +IE_GPU.onnx_model_atanh +IE_GPU.onnx_model_sinh +IE_GPU.onnx_model_cosh +IE_GPU.onnx_model_sign +IE_GPU.onnx_model_shrink_float +IE_GPU.onnx_model_shrink_int +IE_GPU.onnx_model_lp_norm_p1 +IE_GPU.onnx_model_reverse_sequence_0_batch_1 +IE_GPU.onnx_model_reverse_sequence_1_batch_0 +IE_GPU.onnx_matmul_float_type +IE_GPU.onnx_model_pad_constant +IE_GPU.onnx_model_reciprocal +IE_GPU.onnx_dyn_shapes_model_atanh_1_3 +IE_GPU.onnx_dyn_shapes_model_atanh_3_2 +IE_GPU.onnx_dyn_shapes_avg_pool_dyn_shape +IE_GPU.onnx_dyn_shapes_max_pool_dyn_shape +IE_GPU.onnx_dyn_shapes_global_avg_pool_dyn_shape +IE_GPU.onnx_dyn_shapes_global_max_pool_dyn_shape +IE_GPU.onnx_roi_align_f32 +IE_GPU.onnx_model_conv2d_strides_padding +IE_GPU.onnx_model_conv2d_strides_no_padding +IE_GPU.onnx_model_conv2d_strides_assymetric_padding +IE_GPU.onnx_model_conv2d_dilation_assymetric_pads_strides +IE_GPU.onnx_model_conv3d_bias +IE_GPU.onnx_model_global_lp_pool_p2 +IE_GPU.onnx_model_global_lp_pool_p3 +IE_GPU.onnx_model_mean_opset1 +IE_GPU.onnx_model_mean +IE_GPU.onnx_model_reduce_l2 +IE_GPU.onnx_model_acosh +IE_GPU.onnx_model_asinh +IE_GPU.onnx_model_hardmax +IE_GPU.onnx_model_lp_norm_p2 +IE_GPU.onnx_model_lp_norm_default +IE_GPU.onnx_model_instance_normalization +IE_GPU.onnx_model_instance_normalization_dyn_shape +IE_GPU.onnx_model_instance_normalization_dyn_shape2 +IE_GPU.onnx_dyn_shapes_model_acosh_1_3 +IE_GPU.onnx_dyn_shapes_model_acosh_3_2 +IE_GPU.onnx_dyn_shapes_model_asinh_1_3 +IE_GPU.onnx_dyn_shapes_model_asinh_3_2 + +# RNN/GRU/LSTM Sequence: Output values mismatch - seq_lengths not supported +IE_GPU.onnx_model_lstm_fwd_mixed_seq_const +IE_GPU.onnx_model_lstm_reverse_mixed_seq_const +IE_GPU.onnx_model_rnn_fwd_mixed_seq_len +IE_GPU.onnx_model_rnn_fwd_mixed_seq_len_const +IE_GPU.onnx_model_gru_fwd_mixed_seq_len +IE_GPU.onnx_model_gru_fwd_mixed_seq_len_const + +IE_GPU.onnx_model_fake_quantize_const_inputs_infer +IE_GPU.onnx_model_fake_quantize_nonconst_inputs_infer + +# Not supported dynamic shapes cases for Loop +onnx_controlflow_loop_2d_no_identity_termination_cond +onnx_controlflow_loop_2d_no_identity_termination_cond_false +onnx_controlflow_loop_2d_trip_count_max_int +onnx_controlflow_loop_2d_const_no_identity_termination_cond +onnx_controlflow_loop_2d_both_cond_and_trip_count_as_inputs +onnx_controlflow_loop_no_variadic_inputs_and_outputs +onnx_controlflow_loop_power +onnx_if_dynamic_inputs + +# Input body shape is changed during Loop iterations +# Exception is throw during Loop shape inference +onnx_controlflow_loop_concat_values + +# dynamic trip count +onnx_controlflow_loop_2d_trip_count_dynamic + +# Infinitive Loop is not supported +onnx_controlflow_loop_infinite + +# unsupported dynamic ops +onnx_dyn_shapes_reduce_max_dynamic_input_rank_negative_axis + +# not implemented yet on GPU plugin +IE_GPU.onnx_model_gather_elements_float_1D +IE_GPU.onnx_model_gather_elements_float_negative_axis +IE_GPU.onnx_model_gather_elements_int32_axis_0 +IE_GPU.onnx_model_gather_elements_int8_axis_1 +IE_GPU.onnx_model_gather_elements_float_3D_axis_2 + +IE_CPU/ElemTypesTests/1.onnx_test_add_abc_set_precission + +# RuntimeError: Unsupported dynamic ops: v4::Interpolate - Ticket: 50691 +onnx_upsample6_dynamic + +# random values returned from the plugin: ticket 51762 +onnx_model_deformable_conv_2d + +# Not yet implemented - refer PR#6601 +IE_CPU.onnx_model_gather_float_2D_neg_indices + +# CPU plug-in doesn't support operation with dynamic rank +onnx_model_skip_layer_normalization_dynamic_shapes + +# Doesn't support op with dynamic shapes +onnx_model_embed_layer_normalization_dynamic_shapes + +# CPU plug-in doesn't support operation with dynamic rank +onnx_model_attention_dynamic_shapes + +# CPU plug-in doesn't support Parameter operation with dynamic rank +IE_CPU.onnx_scan15_dyn_rank_vals + +# z node not found in graph cache ticket: 81976 +IE_CPU.onnx_expand_context_dependent_function +IE_CPU.onnx_softmax_crossentropy_loss_mean diff --git a/src/frontends/tests/onnx/runtime/interpreter/unit_test.manifest b/src/frontends/tests/onnx/runtime/interpreter/unit_test.manifest new file mode 100644 index 00000000000..53819b14a8d --- /dev/null +++ b/src/frontends/tests/onnx/runtime/interpreter/unit_test.manifest @@ -0,0 +1,84 @@ +INTERPRETER.onnx_top_k_opset_10 + +# Temporarily disabled: +INTERPRETER.onnx_resize11_scales_nearest_asymmetric_floor_dynamic_sizes +INTERPRETER.onnx_resize11_up_sizes_cubic_half_pixel_dynamic_sizes +INTERPRETER.onnx_resize10_up_scales_const_nearest +INTERPRETER.onnx_resize10_up_scales_const_linear +INTERPRETER.onnx_resize10_down_scales_const_nearest + +# Failed in MacOS: +INTERPRETER.onnx_resize11_sizes_nearest_asymmetric_floor + +# Disabled tests for disabled reference implementations +INTERPRETER.onnx_dyn_shapes_expand_uint16_dyn_shape + +# Revise reference implementation +onnx_dyn_model_hardmax +onnx_model_quantize_linear_const_scale_const_zero_p +onnx_model_quantize_linear +onnx_model_quantize_linear_axis_zero +onnx_model_quantize_linear_axis_negative + +# Removed opset0 operations +INTERPRETER.onnx_model_dequantize_linear +INTERPRETER.onnx_model_dequantize_linear_scalar_zero_scale_uint8 +INTERPRETER.onnx_model_dequantize_linear_scalar_zero_scale_int8 +INTERPRETER.onnx_model_dequantize_linear_1d_zero_scale_uint8 +INTERPRETER.onnx_model_dequantize_linear_1d_zero_scale_int8 +INTERPRETER.onnx_model_dequantize_linear_1d_zero_scale_int8_4d +INTERPRETER.onnx_model_dequantize_linear_1d_zero_scale_uint8_negative_axis + +# ONNX evaluate method not implemented for If operator +INTERPRETER.onnx_if_inside_if +INTERPRETER.onnx_if_inside_loop + +# Activation function hardsigmoid unsupported +onnx_model_gru_fwd_activations_relu_hardsigmoid +onnx_model_lstm_fwd_hardsigmoid_activation + +# Issue 37473. Fails on ia32 platforms only +onnx_model_softmax_axis_0 +onnx_model_reshape_negative_dim + +# LogSoftmax's reference implementation doesn't handle scalar input properly +onnx_model_logsoftmax_0D +# Input body shape is changed during Loop iterations +# Exception is throw during Loop shape inference +onnx_controlflow_loop_concat_values + +# Infinitive Loop is not supported +onnx_controlflow_loop_infinite + +# Dynamic shape support +onnx_controlflow_loop_2d_trip_count_dynamic +onnx_controlflow_loop_no_variadic_inputs_and_outputs +onnx_controlflow_loop_power + +# Interpreter backend doesn't implement evaluate method for OP ExperimentalDetectronROIFeatureExtractor +# INTERPRETER.onnx_model_experimental_detectron_roi_feature_extractor + +# No evaluator for DeformableConv2D +onnx_model_deformable_conv_2d + +# New fails +onnx_model_quant_conv_linear_3d + +INTERPRETER.onnx_dyn_shapes_ab_plus_c_inference +INTERPRETER.onnx_dyn_shapes_dynamic_rank_input_inference +INTERPRETER.onnx_dyn_shapes_model_conv_with_dynamic_bias +INTERPRETER.onnx_dyn_shapes_model_convtranspose_dyn_filters +INTERPRETER.onnx_dyn_shapes_flatten_axis_0 +INTERPRETER.onnx_dyn_shapes_flatten_axis +INTERPRETER.onnx_dyn_shapes_flatten_neg_axis +INTERPRETER.onnx_dyn_shapes_slice_10_slice_2d_default_steps_dyn_begin_end +INTERPRETER.onnx_model_instance_normalization_dyn_shape +INTERPRETER.onnx_controlflow_loop_2d_no_identity_termination_cond_false + +# new failures after fixing the TestCase class - 77385 +quant_dequant_pattern_axis +onnx_clip_no_min_no_max_int64 + +# z node not found in graph cache - ticket: 81976 +INTERPRETER.onnx_expand_context_dependent_function +INTERPRETER.onnx_softmax_crossentropy_loss_mean diff --git a/src/core/tests/onnx_test_util/CMakeLists.txt b/src/frontends/tests/onnx/test_util/CMakeLists.txt similarity index 100% rename from src/core/tests/onnx_test_util/CMakeLists.txt rename to src/frontends/tests/onnx/test_util/CMakeLists.txt diff --git a/src/core/tests/onnx_test_util/onnx_test_util.cpp b/src/frontends/tests/onnx/test_util/onnx_test_util.cpp similarity index 100% rename from src/core/tests/onnx_test_util/onnx_test_util.cpp rename to src/frontends/tests/onnx/test_util/onnx_test_util.cpp diff --git a/src/core/tests/onnx_test_util/onnx_test_util.hpp b/src/frontends/tests/onnx/test_util/onnx_test_util.hpp similarity index 100% rename from src/core/tests/onnx_test_util/onnx_test_util.hpp rename to src/frontends/tests/onnx/test_util/onnx_test_util.hpp