From 24b414c16862822fac74642909b026c98c71b33b Mon Sep 17 00:00:00 2001 From: Andrey Kashchikhin Date: Sat, 16 Dec 2023 08:37:37 +0000 Subject: [PATCH] [CI] [GHA] Introduce ONNX Models Tests job in Linux workflow (#21581) * add onnx models test job * rm sudo * add missing update * add missing git-lfs * add missing package * correct var * correct command * try with pytest * provide path to models * use mounted path * add msft models * use symlinks * correct name for model * use -h * create and use directory for models * use share as source for models * fix test names, use fresh onnx repo * return names * fix test names * change folder structure for onnx model downloading * skip tests; clean up logs; adapt azure counterpart * make skip names lowercase * increase parallel * add missing git-lfs * fix dir * install git-lfs properly * use sudo * use 12 parallels; rm -f * use skip * rm unnecessary requirements * rm pytorch from installing * disable test --- .ci/azure/linux_ngraph_onnx.yml | 10 +- .github/workflows/job_onnx_models_tests.yml | 106 ++++++++++++++++++ .github/workflows/linux.yml | 10 ++ src/frontends/onnx/tests/__init__.py | 4 + src/frontends/onnx/tests/onnx_import.in.cpp | 5 + .../tests_python/model_zoo_preprocess.sh | 4 +- .../tests/tests_python/test_zoo_models.py | 31 ++--- .../tests_python/utils/model_importer.py | 2 + 8 files changed, 152 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/job_onnx_models_tests.yml diff --git a/.ci/azure/linux_ngraph_onnx.yml b/.ci/azure/linux_ngraph_onnx.yml index b756eb59149..af8fc3e34f9 100644 --- a/.ci/azure/linux_ngraph_onnx.yml +++ b/.ci/azure/linux_ngraph_onnx.yml @@ -87,7 +87,7 @@ jobs: sudo apt --assume-yes update && sudo apt --assume-yes install nfs-common sudo apt install nfs-common -y sudo mount -vvv -t nfs cinfsshare.file.core.windows.net:/cinfsshare/onnxtestdata $(MODELS_DIR) -o vers=4,minorversion=1,sec=sys - mkdir -p $(MODELS_DIR)/models_data + mkdir -p $(MODELS_DIR) displayName: 'Make dirs' - checkout: self @@ -97,13 +97,15 @@ jobs: - script: | set -e - apt-get update && apt-get install -y lsb-release && apt-get clean all + sudo apt-get update && sudo apt-get install -y lsb-release git git-lfs + git-lfs install + sudo apt-get clean all curl -fsSL https://get.docker.com -o get-docker.sh sudo sh get-docker.sh displayName: 'Install dependencies' - script: - src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh -d $(MODELS_DIR)/models_data -o -s "$(ONNX_MODEL_ZOO_SHA)" + src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh -d $(MODELS_DIR) -o -s "$(ONNX_MODEL_ZOO_SHA)" displayName: 'Update models' condition: ne(variables['BUILD_TYPE'], 'Debug') @@ -120,7 +122,7 @@ jobs: - script: | sudo docker run \ --name openvino-onnx-ci-container \ - --volume $(MODELS_DIR)/models_data/model_zoo/onnx_model_zoo_$(ONNX_MODEL_ZOO_SHA):/root/.onnx/model_zoo/onnx_model_zoo \ + --volume $(MODELS_DIR)/onnx_model_zoo:/root/.onnx/model_zoo/onnx_model_zoo \ --volume $(MODELS_DIR)/msft:/root/.onnx/model_zoo/MSFT openvino-onnx-ci-image \ /bin/bash -c "$(TOX_COMMAND)" displayName: 'Docker run $(BUILD_TYPE)' diff --git a/.github/workflows/job_onnx_models_tests.yml b/.github/workflows/job_onnx_models_tests.yml new file mode 100644 index 00000000000..1fbc0c11fe9 --- /dev/null +++ b/.github/workflows/job_onnx_models_tests.yml @@ -0,0 +1,106 @@ +name: ONNX Models tests + +on: + workflow_call: + inputs: + runner: + description: 'Machine on which the tests would run' + type: string + required: true + container: + description: 'JSON to be converted to the value of the "container" configuration for the job' + type: string + required: false + default: '{"image": null}' + +jobs: + ONNX_Models_tests: + name: ONNX Models tests + timeout-minutes: 60 + runs-on: ${{ inputs.runner }} + container: ${{ fromJSON(inputs.container) }} + defaults: + run: + shell: bash + env: + DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input + OPENVINO_REPO: ${{ github.workspace }}/openvino + INSTALL_DIR: ${{ github.workspace }}/install + INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests + ONNX_MODELS_PATH: ${{ github.workspace }}/onnx_test_models + MODELS_SHARE_PATH: "/mount/onnxtestdata" + ONNX_MODEL_ZOO_SHA: "d58213534f2a4d1c4b19ba62b3bb5f544353256e" + steps: + - name: Download OpenVINO package + uses: actions/download-artifact@v3 + with: + name: openvino_package + path: ${{ env.INSTALL_DIR }} + + - name: Download OpenVINO tests package + uses: actions/download-artifact@v3 + with: + name: openvino_tests + path: ${{ env.INSTALL_TEST_DIR }} + + # Needed as ${{ github.workspace }} is not working correctly when using Docker + - name: Setup Variables + run: | + echo "OPENVINO_REPO=$GITHUB_WORKSPACE/openvino" >> "$GITHUB_ENV" + echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV" + echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/tests" >> "$GITHUB_ENV" + + - name: Extract OpenVINO packages + run: | + pushd ${INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} + popd + + pushd ${INSTALL_TEST_DIR} + tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} + popd + + - name: Fetch setup_python action and model_zoo_preprocess script + uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/actions/setup_python/action.yml + src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh + sparse-checkout-cone-mode: false + path: 'openvino' + + - name: Install dependencies + run: | + # install git (required to build pip deps from the sources) + apt-get update && apt-get install --assume-yes --no-install-recommends git ca-certificates git-lfs + + - name: Setup Python 3.11 + uses: ./openvino/.github/actions/setup_python + with: + version: '3.11' + should-setup-pip-paths: 'false' + self-hosted-runner: ${{ contains(inputs.runner, 'aks') }} + + - name: Update Models + run: bash ${OPENVINO_REPO}/src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh -d ${MODELS_SHARE_PATH} -o -s "${{ env.ONNX_MODEL_ZOO_SHA }}" + + - name: Install OpenVINO Python wheels + run: | + # Install the core OV wheel + python3 -m pip install ${INSTALL_DIR}/tools/openvino-*.whl + + extras_to_install="onnx" + + # Find and install OV dev wheel + pushd ${INSTALL_DIR}/tools + ov_dev_wheel_name=$(find . -name 'openvino_dev*.whl') + python3 -m pip install $ov_dev_wheel_name[$extras_to_install] + popd + + - name: Install Python tests dependencies + run: | + # To enable pytest parallel features + python3 -m pip install pytest-xdist[psutil] pytest-forked + + - name: ONNX Models Tests + run: python3 -m pytest --backend="CPU" --model_zoo_dir="${MODELS_SHARE_PATH}" ${INSTALL_TEST_DIR}/onnx/tests/tests_python/test_zoo_models.py -v -n 12 --forked -k 'not _cuda' --model_zoo_xfail diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index eec0101359a..9cbaec45ec5 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -388,6 +388,16 @@ jobs: container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' sccache-azure-key-prefix: 'ubuntu20_x86_64_onnxruntime' + ONNX_Models: + name: ONNX Models Tests + if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test || + fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE.test + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_onnx_models_tests.yml + with: + runner: 'aks-linux-16-cores-32gb' + container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount:/mount"]}' + CXX_Unit_Tests: name: C++ unit tests needs: [ Build, Smart_CI ] diff --git a/src/frontends/onnx/tests/__init__.py b/src/frontends/onnx/tests/__init__.py index 48d058df016..6db6add7b6a 100644 --- a/src/frontends/onnx/tests/__init__.py +++ b/src/frontends/onnx/tests/__init__.py @@ -168,3 +168,7 @@ xfail_issue_122776 = xfail_test(reason="test_mish_expanded_cpu - " "Not equal to tolerance") xfail_issue_122775 = xfail_test(reason="test_resize_downsample_scales_linear_cpu - " "Not equal to tolerance") + +skip_issue_127649 = pytest.mark.skip(reason="Not equal to tolerance rtol=0.001, atol=1e-07 - " + "Mismatched elements: 1 / 1000 (0.1%)") + diff --git a/src/frontends/onnx/tests/onnx_import.in.cpp b/src/frontends/onnx/tests/onnx_import.in.cpp index dc302f500c4..94626496aca 100644 --- a/src/frontends/onnx/tests/onnx_import.in.cpp +++ b/src/frontends/onnx/tests/onnx_import.in.cpp @@ -1108,7 +1108,12 @@ OPENVINO_TEST(${BACKEND_NAME}, onnx_model_nonmaxsuppression_v9_single_box) { test_case.run(); } +#if (defined _WIN32 || defined _WIN64) +// Ticket: 127743 +OPENVINO_TEST(${BACKEND_NAME}, DISABLED_onnx_model_nonmaxsuppression_default_score_threshold) { +#else OPENVINO_TEST(${BACKEND_NAME}, onnx_model_nonmaxsuppression_default_score_threshold) { +#endif auto function = onnx_import::import_onnx_model(file_util::path_join(ov::test::utils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/nms_default_score_threshold.onnx")); diff --git a/src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh b/src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh index 8181bc77974..19e0dec1577 100755 --- a/src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh +++ b/src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh @@ -56,8 +56,8 @@ while getopts "homfd:s:" opt; do done shift $((OPTIND -1)) -MODEL_ZOO_DIR="$MODELS_DIR/model_zoo" -ONNX_MODELS_DIR="$MODEL_ZOO_DIR/onnx_model_zoo_$ONNX_SHA" +MODEL_ZOO_DIR="$MODELS_DIR" +ONNX_MODELS_DIR="$MODEL_ZOO_DIR/onnx_model_zoo" MSFT_MODELS_DIR="$MODEL_ZOO_DIR/MSFT" function pull_and_postprocess_onnx_model_zoo() { diff --git a/src/frontends/onnx/tests/tests_python/test_zoo_models.py b/src/frontends/onnx/tests/tests_python/test_zoo_models.py index 3388b47d29d..20ddf94cc4b 100644 --- a/src/frontends/onnx/tests/tests_python/test_zoo_models.py +++ b/src/frontends/onnx/tests/tests_python/test_zoo_models.py @@ -22,7 +22,8 @@ from tests import ( xfail_issue_58676, xfail_issue_78843, xfail_issue_86911, - xfail_issue_onnx_models_140) + xfail_issue_onnx_models_140, + skip_issue_127649) MODELS_ROOT_DIR = tests.MODEL_ZOO_DIR @@ -162,7 +163,7 @@ if len(zoo_models) > 0: (xfail_issue_38701, "test_onnx_model_zoo_text_machine_comprehension_bidirectional_attention_flow_model_bidaf_9_bidaf_bidaf_cpu"), # Model MSFT - (xfail_issue_37957, "test_MSFT_opset10_mask_rcnn_keras_mask_rcnn_keras_cpu"), + (xfail_issue_37957, "test_msft_opset10_mask_rcnn_keras_mask_rcnn_keras_cpu"), ] for test_case in import_xfail_list: xfail, test_name = test_case @@ -178,24 +179,26 @@ if len(zoo_models) > 0: (xfail_issue_39669, "test_onnx_model_zoo_text_machine_comprehension_t5_model_t5_decoder_with_lm_head_12_t5_decoder_with_lm_head_cpu"), (xfail_issue_48145, "test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_8_download_sample_8_bertsquad8_cpu"), (xfail_issue_48190, "test_onnx_model_zoo_text_machine_comprehension_roberta_model_roberta_base_11_roberta_base_11_roberta_base_11_cpu"), - (xfail_issue_onnx_models_140, "test_onnx_model_zoo_vision_object_detection_segmentation_duc_model_ResNet101_DUC_7_ResNet101_DUC_HDC_ResNet101_DUC_HDC_cpu"), + (xfail_issue_onnx_models_140, "test_onnx_model_zoo_vision_object_detection_segmentation_duc_model_resnet101_duc_7_resnet101_duc_hdc_resnet101_duc_hdc_cpu"), (xfail_issue_78843, "test_onnx_model_zoo_vision_object_detection_segmentation_ssd_mobilenetv1_model_ssd_mobilenet_v1_10_ssd_mobilenet_v1_ssd_mobilenet_v1_cpu"), + (skip_issue_127649, "test_onnx_model_zoo_vision_classification_resnet_model_resnet50_v1_7_resnet50v1_resnet50_v1_7_cpu"), + (skip_issue_127649, "test_onnx_model_zoo_vision_super_resolution_sub_pixel_cnn_2016_model_super_resolution_10_super_resolution_super_resolution_cpu"), # Model MSFT - (xfail_issue_37973, "test_MSFT_opset7_tf_inception_v2_model_cpu"), - (xfail_issue_37973, "test_MSFT_opset8_tf_inception_v2_model_cpu"), - (xfail_issue_37973, "test_MSFT_opset9_tf_inception_v2_model_cpu"), - (xfail_issue_37973, "test_MSFT_opset11_tf_inception_v2_model_cpu"), - (xfail_issue_37973, "test_MSFT_opset10_tf_inception_v2_model_cpu"), + (xfail_issue_37973, "test_msft_opset7_tf_inception_v2_model_cpu"), + (xfail_issue_37973, "test_msft_opset8_tf_inception_v2_model_cpu"), + (xfail_issue_37973, "test_msft_opset9_tf_inception_v2_model_cpu"), + (xfail_issue_37973, "test_msft_opset11_tf_inception_v2_model_cpu"), + (xfail_issue_37973, "test_msft_opset10_tf_inception_v2_model_cpu"), - (xfail_issue_58676, "test_MSFT_opset7_fp16_tiny_yolov2_onnxzoo_winmlperf_tiny_yolov2_cpu"), - (xfail_issue_58676, "test_MSFT_opset8_fp16_tiny_yolov2_onnxzoo_winmlperf_tiny_yolov2_cpu"), + (xfail_issue_58676, "test_msft_opset7_fp16_tiny_yolov2_onnxzoo_winmlperf_tiny_yolov2_cpu"), + (xfail_issue_58676, "test_msft_opset8_fp16_tiny_yolov2_onnxzoo_winmlperf_tiny_yolov2_cpu"), - (xfail_issue_39669, "test_MSFT_opset9_cgan_cgan_cpu"), - (xfail_issue_47495, "test_MSFT_opset10_BERT_Squad_bertsquad10_cpu"), - (xfail_issue_78843, "test_MSFT_opset10_mlperf_ssd_mobilenet_300_ssd_mobilenet_v1_coco_2018_01_28_cpu"), + (xfail_issue_39669, "test_msft_opset9_cgan_cgan_cpu"), + (xfail_issue_47495, "test_msft_opset10_bert_squad_bertsquad10_cpu"), + (xfail_issue_78843, "test_msft_opset10_mlperf_ssd_mobilenet_300_ssd_mobilenet_v1_coco_2018_01_28_cpu"), - (xfail_issue_86911, "test_MSFT_opset9_LSTM_Seq_lens_unpacked_model_cpu"), + (xfail_issue_86911, "test_msft_opset9_lstm_seq_lens_unpacked_model_cpu"), ] for test_case in import_xfail_list + execution_xfail_list: diff --git a/src/frontends/onnx/tests/tests_python/utils/model_importer.py b/src/frontends/onnx/tests/tests_python/utils/model_importer.py index bacc0a7ab83..e27e10d1ada 100644 --- a/src/frontends/onnx/tests/tests_python/utils/model_importer.py +++ b/src/frontends/onnx/tests/tests_python/utils/model_importer.py @@ -59,6 +59,8 @@ class ModelImportRunner(onnx.backend.test.BackendTest): else: test_name = test_name.replace(string, "_") + test_name = test_name.lower() + test_case = ExtOnnxTestCase( name=test_name, url=None,