diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f7d225d0758..cc2b3eb9b9c 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -369,7 +369,7 @@ jobs: uses: actions/upload-artifact@v3 if: ${{ !cancelled() }} with: - name: test-results-cpp + name: test-results-samples path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml if-no-files-found: 'error' diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml new file mode 100644 index 00000000000..6a8fc2f63c1 --- /dev/null +++ b/.github/workflows/mac.yml @@ -0,0 +1,660 @@ +name: macOS (macOS 12, Python 3.11) +on: + workflow_dispatch: +# pull_request: +# paths-ignore: +# - '**/docs/**' +# - 'docs/**' +# - '**/**.md' +# - '**.md' +# - '**/layer_tests_summary/**' +# - '**/conformance/**' + push: + paths-ignore: + - '**/docs/**' + - 'docs/**' + - '**/**.md' + - '**.md' + - '**/layer_tests_summary/**' + - '**/conformance/**' + branches: + - master + - 'releases/**' + +concurrency: + # github.ref is not unique in post-commit + group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-mac-main + cancel-in-progress: true + +env: + PYTHON_VERSION: '3.11' + +jobs: + Build: + defaults: + run: + shell: bash + runs-on: macos-12-large + env: + CMAKE_BUILD_TYPE: 'Release' + CMAKE_GENERATOR: 'Ninja Multi-Config' + MACOSX_DEPLOYMENT_TARGET: '10.12' + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache + OPENVINO_REPO: ${{ github.workspace }}/openvino + OPENVINO_CONTRIB_REPO: ${{ github.workspace }}/openvino_contrib + INSTALL_DIR: ${{ github.workspace }}/openvino_install + INSTALL_TEST_DIR: ${{ github.workspace }}/tests_install + BUILD_DIR: ${{ github.workspace }}/build + steps: + - name: Clone OpenVINO + uses: actions/checkout@v4 + with: + path: 'openvino' + submodules: 'true' + + - name: Clone OpenVINO Contrib + uses: actions/checkout@v4 + with: + repository: 'openvinotoolkit/openvino_contrib' + path: 'openvino_contrib' + + # + # Dependencies + # + + - name: Install build dependencies + run: brew install coreutils ninja scons + + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install python dependencies + run: | + # For Python API + python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/wheel/requirements-dev.txt + python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/requirements.txt + + # For running Python API tests + python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/src/compatibility/openvino/requirements-dev.txt + + # For running ONNX frontend unit tests + python3 -m pip install --force-reinstall -r ${{ env.OPENVINO_REPO }}/src/frontends/onnx/tests/requirements.txt + + # For running TensorFlow frontend unit tests + python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/tensorflow/tests/requirements.txt + + # For running Paddle frontend unit tests + python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/paddle/tests/requirements.txt + + # + # Build + # + + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + max-size: "2000M" + # Should save cache only if run in the master branch of the base repo + # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push + save: ${{ github.ref_name == 'master' && 'true' || 'false' }} + verbose: 2 + key: ${{ runner.os }}-main + restore-keys: | + ${{ runner.os }}-main + + - name: CMake configure + run: | + cmake \ + -G "${{ env.CMAKE_GENERATOR }}" \ + -DENABLE_CPPLINT=OFF \ + -DENABLE_NCC_STYLE=OFF \ + -DENABLE_TESTS=ON \ + -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \ + -DENABLE_STRICT_DEPENDENCIES=OFF \ + -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CMAKE_CXX_COMPILER_LAUNCHER }} \ + -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \ + -S ${{ env.OPENVINO_REPO }} \ + -B ${{ env.BUILD_DIR }} + + - name: Cmake build - OpenVINO + run: cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + + - name: Show ccache stats + run: ccache --show-stats + + - name: Cmake install - OpenVINO + run: | + cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -P ${{ env.BUILD_DIR }}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_TEST_DIR }} -DCOMPONENT=tests -P ${{ env.BUILD_DIR }}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -DCOMPONENT=python_wheels -P ${{ env.BUILD_DIR }}/cmake_install.cmake + + - name: Pack Artifacts + run: | + pushd ${{ env.INSTALL_DIR }} + tar -czvf ${{ env.BUILD_DIR }}/openvino_package.tar.gz * + popd + + pushd ${{ env.INSTALL_TEST_DIR }} + tar -czvf ${{ env.BUILD_DIR }}/openvino_tests.tar.gz * + popd + + - name: Cmake & Build - OpenVINO Contrib + run: | + cmake \ + -DBUILD_nvidia_plugin=OFF \ + -DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose" \ + -DOPENVINO_EXTRA_MODULES=${{ env.OPENVINO_CONTRIB_REPO }}/modules \ + -S ${{ env.OPENVINO_REPO }} \ + -B ${{ env.BUILD_DIR }} + cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + + # + # Upload build artifacts + # + + - name: Upload openvino package + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: openvino_package + path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz + if-no-files-found: 'error' + + - name: Upload openvino tests package + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: openvino_tests + path: ${{ env.BUILD_DIR }}/openvino_tests.tar.gz + if-no-files-found: 'error' + + Samples: + needs: Build + defaults: + run: + shell: bash + runs-on: macos-12 + env: + INSTALL_DIR: ${{ github.workspace }}/install + INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests + BUILD_DIR: ${{ github.workspace }}/build + + steps: + # + # Initialize OpenVINO + # + + - name: Download OpenVINO package + uses: actions/download-artifact@v3 + with: + name: openvino_package + path: ${{ env.INSTALL_DIR }} + + - name: Download OpenVINO tests package + uses: actions/download-artifact@v3 + with: + name: openvino_tests + path: ${{ env.INSTALL_TEST_DIR }} + + - name: Extract OpenVINO packages + run: | + pushd ${INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} + popd + + pushd ${INSTALL_TEST_DIR} + tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} + popd + + - name: Install dependencies + run: brew install coreutils + + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Build cpp samples + run: ${INSTALL_DIR}/samples/cpp/build_samples.sh -i ${INSTALL_DIR} -b ${BUILD_DIR}/cpp_samples + env: + CMAKE_COMPILE_WARNING_AS_ERROR: 'ON' + + - name: Build c samples + run: ${INSTALL_DIR}/samples/c/build_samples.sh -i ${INSTALL_DIR} -b ${BUILD_DIR}/c_samples + env: + CMAKE_COMPILE_WARNING_AS_ERROR: 'ON' + + # + # Tests + # + + - name: Samples tests + run: | + export WORKSPACE=${INSTALL_DIR} + export IE_APP_PATH=${INSTALL_DIR}/samples_bin + export IE_APP_PYTHON_PATH=${INSTALL_DIR}/samples/python + export SHARE=${INSTALL_TEST_DIR}/smoke_tests/samples_smoke_tests_data + + python3 -m pip install --ignore-installed PyYAML -r ${INSTALL_TEST_DIR}/smoke_tests/requirements.txt + + source ${INSTALL_DIR}/setupvars.sh + + python3 -m pytest -sv ${INSTALL_TEST_DIR}/smoke_tests \ + --ignore=${INSTALL_TEST_DIR}/smoke_tests/test_speech_sample.py \ + --env_conf ${INSTALL_TEST_DIR}/smoke_tests/env_config.yml \ + --junitxml=${INSTALL_TEST_DIR}/TEST-SamplesSmokeTests.xml + + - name: Upload Test Results + uses: actions/upload-artifact@v3 + if: ${{ !cancelled() }} + with: + name: test-results-samples + path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml + if-no-files-found: 'error' + + CXX_Unit_Tests: + name: C++ Unit tests + needs: Build + defaults: + run: + shell: bash + runs-on: macos-12 + env: + INSTALL_DIR: ${{ github.workspace }}/install + INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests + + steps: + # + # Dependencies + # + + - name: Download OpenVINO package + uses: actions/download-artifact@v3 + with: + name: openvino_package + path: ${{ env.INSTALL_DIR }} + + - name: Download OpenVINO tests package + uses: actions/download-artifact@v3 + with: + name: openvino_tests + path: ${{ env.INSTALL_TEST_DIR }} + + - name: Extract OpenVINO packages + run: | + pushd ${{ env.INSTALL_DIR }} + tar -xzf openvino_package.tar.gz -C ${{ env.INSTALL_DIR }} && rm openvino_package.tar.gz || exit 1 + popd + pushd ${{ env.INSTALL_TEST_DIR }} + tar -xzf openvino_tests.tar.gz -C ${{ env.INSTALL_DIR }} && rm openvino_tests.tar.gz || exit 1 + popd + + # + # Tests + # + + - name: OpenVINO Core Unit Tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-NGraphUT.xml + + - name: OpenVINO Inference Functional Tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_inference_functional_tests --gtest_print_time=1 \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-InferenceFunc.xml + + - name: OpenVINO Inference Unit Tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_inference_unit_tests --gtest_print_time=1 \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-InferenceUnit.xml + + - name: Low Precision Transformations Tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_lp_transformations_tests --gtest_print_time=1 \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-LpTransformations.xml + + - name: OpenVINO Conditional compilation tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_conditional_compilation_tests --gtest_print_time=1 \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ConditionalCompilation.xml + + - name: IR frontend tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_ir_frontend_tests --gtest_print_time=1 \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-IRFrontend.xml + + - name: PaddlePaddle frontend tests + if: ${{ 'false' }} + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/paddle_tests --gtest_print_time=1 --gtest_filter=*smoke* \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-PaddleTests.xml + + - name: ONNX frontend tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_onnx_frontend_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ONNXFrontend.xml + + - name: TensorFlow Common tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_tensorflow_common_tests --gtest_print_time=1 \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-TensorFlowCommonFrontend.xml + + - name: TensorFlow frontend tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_tensorflow_frontend_tests --gtest_print_time=1 \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-TensorFlowFrontend.xml + + - name: TensorFlow Lite frontend tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_tensorflow_lite_frontend_tests --gtest_print_time=1 \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-TensorFlowLiteFrontend.xml + + - name: Transformations Tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_transformations_tests --gtest_print_time=1 \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-Transformations.xml + + - name: Common test utils tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_util_tests --gtest_print_time=1 \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-commonUtilsTests.xml + + - name: CPU plugin unit tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_cpu_unit_tests --gtest_print_time=1 \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-CPUUnitTests.xml + + - name: AUTO unit tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_auto_unit_tests --gtest_print_time=1 \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_unit_tests.xml + + - name: Template plugin tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_template_func_tests --gtest_print_time=1 \ + --gtest_filter=*smoke* \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-TemplateFuncTests.xml + + - name: Inference Engine C API tests + if: ${{ always() }} + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/InferenceEngineCAPITests --gtest_print_time=1 \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-InferenceEngineCAPITests.xml + + - name: OpenVINO C API tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_capi_test --gtest_print_time=1 \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OpenVINOCAPITests.xml + + - name: AutoBatch func tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_auto_batch_func_tests --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_batch_func_tests.xml + + - name: Upload Test Results + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: test-results-cpp + path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml + if-no-files-found: 'error' + + Python_Unit_Tests: + name: Python unit tests + needs: Build + if: ${{ always() }} + defaults: + run: + shell: bash + runs-on: macos-12 + env: + OPENVINO_REPO: ${{ github.workspace }}/openvino + OPENVINO_CONTRIB_REPO: ${{ github.workspace }}/openvino_contrib + INSTALL_DIR: ${{ github.workspace }}/install + INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests + LAYER_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/layer_tests + steps: + - name: Clone OpenVINO + uses: actions/checkout@v4 + with: + path: 'openvino' + submodules: 'true' + + # + # Dependencies + # + + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Download OpenVINO package + uses: actions/download-artifact@v3 + with: + name: openvino_package + path: ${{ env.INSTALL_DIR }} + + - name: Download OpenVINO tests package + uses: actions/download-artifact@v3 + with: + name: openvino_tests + path: ${{ env.INSTALL_TEST_DIR }} + + - name: Extract OpenVINO packages + run: | + pushd ${{ env.INSTALL_DIR }} + tar -xzf openvino_package.tar.gz -C ${{ env.INSTALL_DIR }} + popd + + pushd ${{ env.INSTALL_TEST_DIR }} + tar -xzf openvino_tests.tar.gz -C ${{ env.INSTALL_DIR }} + popd + + - name: Install Python API tests dependencies + run: | + # For torchvision to OpenVINO preprocessing converter + python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/src/openvino/preprocess/torchvision/requirements.txt + + # TODO: replace with Python API tests requirements + python3 -m pip install -r ${{ env.OPENVINO_REPO }}/tools/mo/requirements_dev.txt + + - name: Install OpenVINO Python wheels + run: | + # Install the core OV wheel + python3 -m pip install ${{ env.INSTALL_DIR }}/tools/openvino-*.whl + + # Find and install OV dev wheel + pushd ${{ env.INSTALL_DIR }}/tools + ov_dev_wheel_name=$(find . -name 'openvino_dev*.whl') + python3 -m pip install $ov_dev_wheel_name[mxnet,caffe,kaldi,onnx,tensorflow2] + popd + + - name: nGraph and IE Python Bindings Tests + if: ${{ always() }} + run: | + python3 -m pytest -s ${{ env.INSTALL_TEST_DIR }}/pyngraph \ + --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-Pyngraph.xml \ + --ignore=${{ env.INSTALL_TEST_DIR }}/pyngraph/tests/test_onnx/test_zoo_models.py \ + --ignore=${{ env.INSTALL_TEST_DIR }}/pyngraph/tests/test_onnx/test_backend.py + + - name: Python API 2.0 Tests + if: ${{ always() }} + run: | + # For python imports to import pybind_mock_frontend + export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}:$PYTHONPATH + # for 'template' extension + export DYLD_LIBRARY_PATH=${{ env.INSTALL_TEST_DIR }}:$DYLD_LIBRARY_PATH + + python3 -m pytest -sv ${{ env.INSTALL_TEST_DIR }}/pyopenvino \ + --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-Pyngraph.xml \ + --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_utils/test_utils.py \ + --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_onnx/test_zoo_models.py \ + --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_onnx/test_backend.py + + - name: MO Python API Tests + if: ${{ always() }} + run: | + python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt + export PYTHONPATH=${{ env.LAYER_TESTS_INSTALL_DIR }}:$PYTHONPATH + # TODO: remove setupvars.sh from here; currently, it's used for 'test_utils' installed in '/python/openvino' + source ${INSTALL_DIR}/setupvars.sh + + python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/mo_python_api_tests/ --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-test_mo_convert.xml + env: + TEST_DEVICE: CPU + TEST_PRECISION: FP16 + + - name: Model Optimizer unit tests + if: ${{ always() }} + run: | + export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}:$PYTHONPATH + python3 -m pytest -s ${{ env.INSTALL_TEST_DIR }}/mo/unit_tests \ + --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-ModelOptimizer.xml + + - name: PyTorch Layer Tests + if: ${{ always() }} + run: | + python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt + export PYTHONPATH=${{ env.LAYER_TESTS_INSTALL_DIR }}:$PYTHONPATH + + python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/pytorch_tests -m precommit --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-pytorch.xml + env: + TEST_DEVICE: CPU + TEST_PRECISION: FP16 + + - name: TensorFlow 1 Layer Tests - TF FE + if: ${{ always() }} + run: | + python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt + + export PYTHONPATH=${{ env.OPENVINO_REPO }}/tools/mo/:${{ env.LAYER_TESTS_INSTALL_DIR }}:$PYTHONPATH + + python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_tests/ --use_new_frontend -m precommit_tf_fe --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf_fe.xml + env: + TEST_DEVICE: CPU + + - name: TensorFlow 2 Layer Tests - TF FE + if: ${{ always() }} + run: | + python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt + export PYTHONPATH=${{ env.OPENVINO_REPO }}/tools/mo/:${{ env.LAYER_TESTS_INSTALL_DIR }}:$PYTHONPATH + + python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow2_keras_tests/ --use_new_frontend -m precommit_tf_fe --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf2_fe.xml + env: + TEST_DEVICE: CPU + + - name: TensorFlow 1 Layer Tests - Legacy FE + if: ${{ always() }} + run: | + python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt + export PYTHONPATH=${{ env.OPENVINO_REPO }}/tools/mo/:${{ env.LAYER_TESTS_INSTALL_DIR }}:$PYTHONPATH + + python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_tests/test_tf_Roll.py --ir_version=10 --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf_Roll.xml + + - name: TensorFlow 2 Layer Tests - Legacy FE + if: ${{ always() }} + run: | + python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt + export PYTHONPATH=${{ env.OPENVINO_REPO }}/tools/mo/:${{ env.LAYER_TESTS_INSTALL_DIR }}:$PYTHONPATH + + python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow2_keras_tests/test_tf2_keras_activation.py \ + --ir_version=11 --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf2_Activation.xml -k "sigmoid" + env: + TEST_DEVICE: CPU + + - name: TensorFlow Lite Layer Tests - TFL FE + if: ${{ always() }} + run: | + python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt + export PYTHONPATH=${{ env.OPENVINO_REPO }}/tools/mo/:${{ env.LAYER_TESTS_INSTALL_DIR }}:$PYTHONPATH + + python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_lite_tests/ --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tfl_fe.xml + env: + TEST_DEVICE: CPU + + - name: Python Frontend tests + if: ${{ always() }} + run: | + python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt + export PYTHONPATH=${{ env.OPENVINO_REPO }}/tools/mo/:${{ env.LAYER_TESTS_INSTALL_DIR }}:$PYTHONPATH + # to allow 'libtest_builtin_extensions.so' to find 'libopenvino_onnx_frontend.so' + source ${{ env.INSTALL_DIR }}/setupvars.sh + + python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/py_frontend_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-test_py_fontend.xml + + - name: Upload Test Results + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: test-results-python + path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml + if-no-files-found: 'error' + +# Ticket: 122001 +# CPU_Functional_Tests: +# needs: Build +# if: ${{ always() }} +# defaults: +# run: +# shell: bash +# runs-on: macos-12 +# env: +# INSTALL_DIR: ${{ github.workspace }}/install +# INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests +# +# steps: +# - name: Create Directories +# run: mkdir -p ${{ env.INSTALL_DIR }} ${{ env.INSTALL_TEST_DIR }} +# +# - name: Download OpenVINO package +# uses: actions/download-artifact@v3 +# with: +# name: openvino_package +# path: ${{ env.INSTALL_DIR }} +# +# - name: Download OpenVINO tests package +# uses: actions/download-artifact@v3 +# with: +# name: openvino_tests +# path: ${{ env.INSTALL_TEST_DIR }} +# +# - name: Extract OpenVINO packages +# run: | +# pushd ${{ env.INSTALL_DIR }} +# tar -xzf openvino_package.tar.gz -C ${{ env.INSTALL_DIR }} && rm openvino_package.tar.gz || exit 1 +# popd +# pushd ${{ env.INSTALL_TEST_DIR }} +# tar -xzf openvino_tests.tar.gz -C ${{ env.INSTALL_DIR }} && rm openvino_tests.tar.gz || exit 1 +# popd +# +# - name: Intel CPU plugin func tests +# run: | +# source ${{ env.INSTALL_DIR }}/setupvars.sh +# ${{ env.INSTALL_TEST_DIR }}/ov_cpu_func_tests --gtest_print_time=1 --gtest_filter=*smoke* --gtest_output=xml:"${{ env.INSTALL_TEST_DIR }}/TEST-CPUFuncTests.xml" +# +# - name: Upload Test Results +# uses: actions/upload-artifact@v3 +# if: ${{ always() }} +# with: +# name: test-results-functional-cpu +# path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml +# if-no-files-found: 'error' diff --git a/src/bindings/python/tests_compatibility/test_onnx/test_backend.py b/src/bindings/python/tests_compatibility/test_onnx/test_backend.py index 87f53223c2d..f9d7c4fe261 100644 --- a/src/bindings/python/tests_compatibility/test_onnx/test_backend.py +++ b/src/bindings/python/tests_compatibility/test_onnx/test_backend.py @@ -32,6 +32,7 @@ from tests_compatibility import ( xfail_issue_48052, xfail_issue_52463, xfail_issue_58033, + xfail_issue_58676, xfail_issue_63033, xfail_issue_63036, xfail_issue_63043, @@ -301,6 +302,10 @@ tests_expected_to_fail = [ "OnnxBackendNodeModelTest.test_tril_zero_cpu", "OnnxBackendNodeModelTest.test_triu_zero_cpu", ), + ( + xfail_issue_58676, + "OnnxBackendNodeModelTest.test_div_uint8_cpu" + ), ( skip_dynamic_model, "OnnxBackendNodeModelTest.test_triu_one_row_cpu", diff --git a/tests/layer_tests/pytorch_tests/test_rfftn_complex_transforms.py b/tests/layer_tests/pytorch_tests/test_rfftn_complex_transforms.py index 7e3dd169e5b..a871bbc46fd 100644 --- a/tests/layer_tests/pytorch_tests/test_rfftn_complex_transforms.py +++ b/tests/layer_tests/pytorch_tests/test_rfftn_complex_transforms.py @@ -1,9 +1,12 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +from sys import platform + import numpy as np import pytest import torch + from pytorch_layer_test_class import PytorchLayerTest @@ -40,7 +43,9 @@ class TestRFFTN(PytorchLayerTest): @pytest.mark.parametrize("norm", ["forward", "backward", "ortho", None]) @pytest.mark.nightly @pytest.mark.precommit + @pytest.mark.skipif(platform == 'darwin', reason="Ticket - 122182") def test_rfftn(self, ie_device, precision, ir_version, input_shape, dim, s, norm): self.input_shape = input_shape # Unfrozen test would fail due to issues with prim::GetAttr containing lists, strings or none. - self._test(*self.create_model(dim, s, norm), ie_device, precision, ir_version, custom_eps=1e-3, freeze_model=True) + self._test(*self.create_model(dim, s, norm), ie_device, precision, ir_version, custom_eps=1e-3, + freeze_model=True) diff --git a/tests/layer_tests/pytorch_tests/test_upsample.py b/tests/layer_tests/pytorch_tests/test_upsample.py index cc862ee6cb5..7406302ef5d 100644 --- a/tests/layer_tests/pytorch_tests/test_upsample.py +++ b/tests/layer_tests/pytorch_tests/test_upsample.py @@ -1,6 +1,8 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +from sys import platform + import pytest from pytorch_layer_test_class import PytorchLayerTest @@ -41,6 +43,7 @@ class TestUpsample1D(PytorchLayerTest): ]) @pytest.mark.nightly @pytest.mark.precommit + @pytest.mark.skipif(platform == 'darwin', reason="Ticket - 122182") def test_upsample1d(self, mode, size, scale, ie_device, precision, ir_version): self._test(*self.create_model(size, scale, mode), ie_device, precision, ir_version, trace_model=True) @@ -96,7 +99,6 @@ class TestUpsample2D(PytorchLayerTest): precision, ir_version, trace_model=True, **{"custom_eps": 1e-3}) - class TestUpsample2DAntialias(PytorchLayerTest): def _prepare_input(self): import numpy as np diff --git a/tests/layer_tests/tensorflow_tests/test_tf_CTCLoss.py b/tests/layer_tests/tensorflow_tests/test_tf_CTCLoss.py index 78289a7144b..0a2eae63033 100644 --- a/tests/layer_tests/tensorflow_tests/test_tf_CTCLoss.py +++ b/tests/layer_tests/tensorflow_tests/test_tf_CTCLoss.py @@ -1,6 +1,8 @@ # Copyright (C) 2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +from sys import platform + import numpy as np import pytest import tensorflow as tf @@ -54,6 +56,7 @@ class TestCTCLoss(CommonTFLayerTest): @pytest.mark.parametrize("params", test_data) @pytest.mark.precommit_tf_fe @pytest.mark.nightly + @pytest.mark.skipif(platform == 'darwin', reason="Ticket - 122182") def test_ctcloss_placeholder_const(self, params, ie_device, precision, ir_version, temp_dir, use_new_frontend, use_old_api): self._test(*self.create_ctcloss_placeholder_const_net(**params), diff --git a/tests/layer_tests/tensorflow_tests/test_tf_FusedBatchNorm.py b/tests/layer_tests/tensorflow_tests/test_tf_FusedBatchNorm.py index 94ae7dc6282..b2d44b5a07b 100644 --- a/tests/layer_tests/tensorflow_tests/test_tf_FusedBatchNorm.py +++ b/tests/layer_tests/tensorflow_tests/test_tf_FusedBatchNorm.py @@ -1,6 +1,8 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +from sys import platform + import numpy as np import pytest import tensorflow as tf @@ -102,6 +104,7 @@ class TestFusedBatchNorm(CommonTFLayerTest): @pytest.mark.parametrize("params", test_data_basic) @pytest.mark.precommit_tf_fe @pytest.mark.nightly + @pytest.mark.skipif(platform == 'darwin', reason="Ticket - 122182") def test_fused_batch_norm_basic(self, params, ie_device, precision, ir_version, temp_dir, use_new_frontend, use_old_api): self._test(*self.create_fused_batch_norm_net(**params), diff --git a/tests/layer_tests/tensorflow_tests/test_tf_TensorListLength.py b/tests/layer_tests/tensorflow_tests/test_tf_TensorListLength.py index 57b17b3341d..8d6005350fb 100644 --- a/tests/layer_tests/tensorflow_tests/test_tf_TensorListLength.py +++ b/tests/layer_tests/tensorflow_tests/test_tf_TensorListLength.py @@ -1,6 +1,8 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +from sys import platform + import numpy as np import pytest import tensorflow as tf @@ -38,6 +40,7 @@ class TestTensorListLength(CommonTFLayerTest): @pytest.mark.parametrize("params", test_data_basic) @pytest.mark.precommit_tf_fe @pytest.mark.nightly + @pytest.mark.skipif(platform == 'darwin', reason="Ticket - 122182") def test_tensor_list_length_basic(self, params, ie_device, precision, ir_version, temp_dir, use_new_frontend, use_old_api): self._test(*self.create_tensor_list_length(**params), @@ -76,6 +79,7 @@ class TestTensorListLengthEmptyList(CommonTFLayerTest): @pytest.mark.parametrize("params", test_data_tensor_list_length_empty_list) @pytest.mark.precommit_tf_fe @pytest.mark.nightly + @pytest.mark.skipif(platform == 'darwin', reason="Ticket - 122182") def test_tensor_list_length_empty_list(self, params, ie_device, precision, ir_version, temp_dir, use_new_frontend, use_old_api): self._test(*self.create_tensor_list_length_empty_list(**params), diff --git a/tests/layer_tests/tensorflow_tests/test_tf_TensorListResize.py b/tests/layer_tests/tensorflow_tests/test_tf_TensorListResize.py index 39bdca06dee..709f009a7af 100644 --- a/tests/layer_tests/tensorflow_tests/test_tf_TensorListResize.py +++ b/tests/layer_tests/tensorflow_tests/test_tf_TensorListResize.py @@ -1,6 +1,8 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +from sys import platform + import numpy as np import pytest import tensorflow as tf @@ -42,6 +44,7 @@ class TestTensorListResize(CommonTFLayerTest): @pytest.mark.parametrize("params", test_data_basic) @pytest.mark.precommit_tf_fe @pytest.mark.nightly + @pytest.mark.skipif(platform == 'darwin', reason="Ticket - 122182") def test_tensor_list_resize_basic(self, params, ie_device, precision, ir_version, temp_dir, use_new_frontend, use_old_api): self._test(*self.create_tensor_list_resize(**params), diff --git a/tests/layer_tests/tensorflow_tests/test_tf_UnaryOps.py b/tests/layer_tests/tensorflow_tests/test_tf_UnaryOps.py index 7b176344271..91ed0a72db0 100644 --- a/tests/layer_tests/tensorflow_tests/test_tf_UnaryOps.py +++ b/tests/layer_tests/tensorflow_tests/test_tf_UnaryOps.py @@ -214,6 +214,7 @@ class TestUnaryOps(CommonTFLayerTest): 'Selu' ]) @pytest.mark.nightly + @pytest.mark.skipif(sys.platform == 'darwin', reason="Ticket - 122182") def test_unary_op(self, params, ie_device, precision, ir_version, temp_dir, op_type, use_new_frontend, use_old_api): if ie_device == 'GPU': diff --git a/tests/layer_tests/tensorflow_tests/test_tf_While.py b/tests/layer_tests/tensorflow_tests/test_tf_While.py index 3a6f9b7f291..2a112700f30 100644 --- a/tests/layer_tests/tensorflow_tests/test_tf_While.py +++ b/tests/layer_tests/tensorflow_tests/test_tf_While.py @@ -1,6 +1,8 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +from sys import platform + import numpy as np import pytest import tensorflow as tf @@ -55,6 +57,7 @@ class TestWhile(CommonTFLayerTest): @pytest.mark.parametrize("params", test_data_basic) @pytest.mark.precommit_tf_fe @pytest.mark.nightly + @pytest.mark.skipif(platform == 'darwin', reason="Ticket - 122182") def test_while_basic(self, params, ie_device, precision, ir_version, temp_dir, use_new_frontend, use_old_api): self._test(*self.create_while_net(**params), @@ -113,6 +116,7 @@ class TestWhileShapeVariant(CommonTFLayerTest): @pytest.mark.parametrize("params", test_data_basic) @pytest.mark.precommit_tf_fe @pytest.mark.nightly + @pytest.mark.skipif(platform == 'darwin', reason="Ticket - 122182") def test_while_basic(self, params, ie_device, precision, ir_version, temp_dir, use_new_frontend, use_old_api): self._test(*self.create_while_net(**params), diff --git a/tools/mo/unit_tests/moc_tf_fe/conversion_basic_models_test.py b/tools/mo/unit_tests/moc_tf_fe/conversion_basic_models_test.py index 3e5c46ecb3a..8d905d8f131 100644 --- a/tools/mo/unit_tests/moc_tf_fe/conversion_basic_models_test.py +++ b/tools/mo/unit_tests/moc_tf_fe/conversion_basic_models_test.py @@ -3,10 +3,10 @@ import os import unittest +from sys import platform import numpy as np from generator import generator, generate - from openvino.runtime import Core from openvino.tools.mo.convert import convert_model @@ -40,7 +40,6 @@ class TestMoFreezePlaceholderTFFE(unittest.TestCase): assert values.dtype == dtype assert np.allclose(values, expected) - @generate( *[ ( @@ -240,6 +239,7 @@ class TestMoFreezePlaceholderTFFE(unittest.TestCase): self.basic("ctc_model_based.pbtxt", None, None, None, None, None, None, True, True, False, False) + @unittest.skipIf(platform == 'darwin', reason="Ticket - 122182") def test_conversion_failure_fallback_use_new_frontend(self): with self.assertRaisesRegex(Exception, "\[TensorFlow Frontend\] Internal error, no translator found for operation\(s\)\: "