[CI] [GHA] Introduce ONNX Models Tests job in Linux workflow (#21581)

* add onnx models test job * rm sudo * add missing update * add missing git-lfs * add missing package * correct var * correct command * try with pytest * provide path to models * use mounted path * add msft models * use symlinks * correct name for model * use -h * create and use directory for models * use share as source for models * fix test names, use fresh onnx repo * return names * fix test names * change folder structure for onnx model downloading * skip tests; clean up logs; adapt azure counterpart * make skip names lowercase * increase parallel * add missing git-lfs * fix dir * install git-lfs properly * use sudo * use 12 parallels; rm -f * use skip * rm unnecessary requirements * rm pytorch from installing * disable test
2023-12-16 08:37:37 +00:00 · 2023-12-16 08:37:37 +00:00 · 24b414c168
commit 24b414c168
parent c8125e7e3b
8 changed files with 152 additions and 20 deletions
--- a/.ci/azure/linux_ngraph_onnx.yml
+++ b/.ci/azure/linux_ngraph_onnx.yml
@ -87,7 +87,7 @@ jobs:
      sudo apt --assume-yes update && sudo apt --assume-yes install nfs-common
      sudo apt install nfs-common -y
      sudo mount -vvv -t nfs cinfsshare.file.core.windows.net:/cinfsshare/onnxtestdata $(MODELS_DIR) -o vers=4,minorversion=1,sec=sys
-      mkdir -p $(MODELS_DIR)/models_data
+      mkdir -p $(MODELS_DIR)
    displayName: 'Make dirs'

  - checkout: self
@ -97,13 +97,15 @@ jobs:

  - script: |
      set -e
-      apt-get update && apt-get install -y lsb-release && apt-get clean all
+      sudo apt-get update && sudo apt-get install -y lsb-release git git-lfs
+      git-lfs install
+      sudo apt-get clean all
      curl -fsSL https://get.docker.com -o get-docker.sh
      sudo sh get-docker.sh
    displayName: 'Install dependencies'

  - script:
-      src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh -d $(MODELS_DIR)/models_data -o -s "$(ONNX_MODEL_ZOO_SHA)"
+      src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh -d $(MODELS_DIR) -o -s "$(ONNX_MODEL_ZOO_SHA)"
    displayName: 'Update models'
    condition: ne(variables['BUILD_TYPE'], 'Debug')

@ -120,7 +122,7 @@ jobs:
  - script: |
      sudo docker run \
        --name openvino-onnx-ci-container \
-        --volume $(MODELS_DIR)/models_data/model_zoo/onnx_model_zoo_$(ONNX_MODEL_ZOO_SHA):/root/.onnx/model_zoo/onnx_model_zoo \
+        --volume $(MODELS_DIR)/onnx_model_zoo:/root/.onnx/model_zoo/onnx_model_zoo \
        --volume $(MODELS_DIR)/msft:/root/.onnx/model_zoo/MSFT openvino-onnx-ci-image \
        /bin/bash -c "$(TOX_COMMAND)"
    displayName: 'Docker run $(BUILD_TYPE)'
--- a/.github/workflows/job_onnx_models_tests.yml
+++ b/.github/workflows/job_onnx_models_tests.yml
@ -0,0 +1,106 @@
+name: ONNX Models tests
+
+on:
+  workflow_call:
+    inputs:
+      runner:
+        description: 'Machine on which the tests would run'
+        type: string
+        required: true
+      container:
+        description: 'JSON to be converted to the value of the "container" configuration for the job'
+        type: string
+        required: false
+        default: '{"image": null}'
+
+jobs:
+  ONNX_Models_tests:
+    name: ONNX Models tests
+    timeout-minutes: 60
+    runs-on: ${{ inputs.runner }}
+    container: ${{ fromJSON(inputs.container) }}
+    defaults:
+      run:
+        shell: bash
+    env:
+      DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
+      OPENVINO_REPO: ${{ github.workspace }}/openvino
+      INSTALL_DIR: ${{ github.workspace }}/install
+      INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
+      ONNX_MODELS_PATH: ${{ github.workspace }}/onnx_test_models
+      MODELS_SHARE_PATH: "/mount/onnxtestdata"
+      ONNX_MODEL_ZOO_SHA: "d58213534f2a4d1c4b19ba62b3bb5f544353256e"
+    steps:
+      - name: Download OpenVINO package
+        uses: actions/download-artifact@v3
+        with:
+          name: openvino_package
+          path: ${{ env.INSTALL_DIR }}
+
+      - name: Download OpenVINO tests package
+        uses: actions/download-artifact@v3
+        with:
+          name: openvino_tests
+          path: ${{ env.INSTALL_TEST_DIR }}
+
+      # Needed as ${{ github.workspace }} is not working correctly when using Docker
+      - name: Setup Variables
+        run: |
+          echo "OPENVINO_REPO=$GITHUB_WORKSPACE/openvino" >> "$GITHUB_ENV"
+          echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV"
+          echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/tests" >> "$GITHUB_ENV"
+
+      - name: Extract OpenVINO packages
+        run: |
+          pushd ${INSTALL_DIR}
+            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
+          popd
+
+          pushd ${INSTALL_TEST_DIR}
+            tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR}
+          popd
+
+      - name: Fetch setup_python action and model_zoo_preprocess script
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            .github/actions/setup_python/action.yml
+            src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh
+          sparse-checkout-cone-mode: false
+          path: 'openvino'
+
+      - name: Install dependencies
+        run: |
+          # install git (required to build pip deps from the sources)
+          apt-get update && apt-get install --assume-yes --no-install-recommends git ca-certificates git-lfs
+
+      - name: Setup Python 3.11
+        uses: ./openvino/.github/actions/setup_python
+        with:
+          version: '3.11'
+          should-setup-pip-paths: 'false'
+          self-hosted-runner: ${{ contains(inputs.runner, 'aks') }}
+
+      - name: Update Models
+        run: bash ${OPENVINO_REPO}/src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh -d ${MODELS_SHARE_PATH} -o -s "${{ env.ONNX_MODEL_ZOO_SHA }}"
+
+      - name: Install OpenVINO Python wheels
+        run: |
+          # Install the core OV wheel
+          python3 -m pip install ${INSTALL_DIR}/tools/openvino-*.whl
+
+          extras_to_install="onnx"
+
+          # Find and install OV dev wheel
+          pushd ${INSTALL_DIR}/tools
+            ov_dev_wheel_name=$(find . -name 'openvino_dev*.whl')
+            python3 -m pip install $ov_dev_wheel_name[$extras_to_install]
+          popd
+
+      - name: Install Python tests dependencies
+        run: |
+          # To enable pytest parallel features
+          python3 -m pip install pytest-xdist[psutil] pytest-forked
+
+      - name: ONNX Models Tests
+        run: python3 -m pytest --backend="CPU" --model_zoo_dir="${MODELS_SHARE_PATH}" ${INSTALL_TEST_DIR}/onnx/tests/tests_python/test_zoo_models.py -v -n 12 --forked -k 'not _cuda' --model_zoo_xfail
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@ -388,6 +388,16 @@ jobs:
      container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}'
      sccache-azure-key-prefix: 'ubuntu20_x86_64_onnxruntime'

+  ONNX_Models:
+    name: ONNX Models Tests
+    if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test ||
+        fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE.test
+    needs: [ Build, Smart_CI ]
+    uses: ./.github/workflows/job_onnx_models_tests.yml
+    with:
+      runner: 'aks-linux-16-cores-32gb'
+      container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount:/mount"]}'
+
  CXX_Unit_Tests:
    name: C++ unit tests
    needs: [ Build, Smart_CI ]
--- a/src/frontends/onnx/tests/init.py
+++ b/src/frontends/onnx/tests/init.py
@ -168,3 +168,7 @@ xfail_issue_122776 = xfail_test(reason="test_mish_expanded_cpu - "
                                       "Not equal to tolerance")
 xfail_issue_122775 = xfail_test(reason="test_resize_downsample_scales_linear_cpu - "
                                       "Not equal to tolerance")
+
+skip_issue_127649 = pytest.mark.skip(reason="Not equal to tolerance rtol=0.001, atol=1e-07 - "
+                                             "Mismatched elements: 1 / 1000 (0.1%)")
+
--- a/src/frontends/onnx/tests/onnx_import.in.cpp
+++ b/src/frontends/onnx/tests/onnx_import.in.cpp
@ -1108,7 +1108,12 @@ OPENVINO_TEST(${BACKEND_NAME}, onnx_model_nonmaxsuppression_v9_single_box) {
    test_case.run();
 }

+#if (defined _WIN32 || defined _WIN64)
+// Ticket: 127743
+OPENVINO_TEST(${BACKEND_NAME}, DISABLED_onnx_model_nonmaxsuppression_default_score_threshold) {
+#else
 OPENVINO_TEST(${BACKEND_NAME}, onnx_model_nonmaxsuppression_default_score_threshold) {
+#endif
    auto function = onnx_import::import_onnx_model(file_util::path_join(ov::test::utils::getExecutableDirectory(),
                                                                        SERIALIZED_ZOO,
                                                                        "onnx/nms_default_score_threshold.onnx"));
--- a/src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh
+++ b/src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh
@ -56,8 +56,8 @@ while getopts "homfd:s:" opt; do
 done
 shift $((OPTIND -1))

-MODEL_ZOO_DIR="$MODELS_DIR/model_zoo"
-ONNX_MODELS_DIR="$MODEL_ZOO_DIR/onnx_model_zoo_$ONNX_SHA"
+MODEL_ZOO_DIR="$MODELS_DIR"
+ONNX_MODELS_DIR="$MODEL_ZOO_DIR/onnx_model_zoo"
 MSFT_MODELS_DIR="$MODEL_ZOO_DIR/MSFT"

 function pull_and_postprocess_onnx_model_zoo() {
--- a/src/frontends/onnx/tests/tests_python/test_zoo_models.py
+++ b/src/frontends/onnx/tests/tests_python/test_zoo_models.py
@ -22,7 +22,8 @@ from tests import (
    xfail_issue_58676,
    xfail_issue_78843,
    xfail_issue_86911,
-    xfail_issue_onnx_models_140)
+    xfail_issue_onnx_models_140,
+    skip_issue_127649)

 MODELS_ROOT_DIR = tests.MODEL_ZOO_DIR

@ -162,7 +163,7 @@ if len(zoo_models) > 0:
            (xfail_issue_38701, "test_onnx_model_zoo_text_machine_comprehension_bidirectional_attention_flow_model_bidaf_9_bidaf_bidaf_cpu"),

            # Model MSFT
-            (xfail_issue_37957, "test_MSFT_opset10_mask_rcnn_keras_mask_rcnn_keras_cpu"),
+            (xfail_issue_37957, "test_msft_opset10_mask_rcnn_keras_mask_rcnn_keras_cpu"),
        ]
        for test_case in import_xfail_list:
            xfail, test_name = test_case
@ -178,24 +179,26 @@ if len(zoo_models) > 0:
            (xfail_issue_39669, "test_onnx_model_zoo_text_machine_comprehension_t5_model_t5_decoder_with_lm_head_12_t5_decoder_with_lm_head_cpu"),
            (xfail_issue_48145, "test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_8_download_sample_8_bertsquad8_cpu"),
            (xfail_issue_48190, "test_onnx_model_zoo_text_machine_comprehension_roberta_model_roberta_base_11_roberta_base_11_roberta_base_11_cpu"),
-            (xfail_issue_onnx_models_140, "test_onnx_model_zoo_vision_object_detection_segmentation_duc_model_ResNet101_DUC_7_ResNet101_DUC_HDC_ResNet101_DUC_HDC_cpu"),
+            (xfail_issue_onnx_models_140, "test_onnx_model_zoo_vision_object_detection_segmentation_duc_model_resnet101_duc_7_resnet101_duc_hdc_resnet101_duc_hdc_cpu"),
            (xfail_issue_78843, "test_onnx_model_zoo_vision_object_detection_segmentation_ssd_mobilenetv1_model_ssd_mobilenet_v1_10_ssd_mobilenet_v1_ssd_mobilenet_v1_cpu"),
+            (skip_issue_127649, "test_onnx_model_zoo_vision_classification_resnet_model_resnet50_v1_7_resnet50v1_resnet50_v1_7_cpu"),
+            (skip_issue_127649, "test_onnx_model_zoo_vision_super_resolution_sub_pixel_cnn_2016_model_super_resolution_10_super_resolution_super_resolution_cpu"),

            # Model MSFT
-            (xfail_issue_37973, "test_MSFT_opset7_tf_inception_v2_model_cpu"),
-            (xfail_issue_37973, "test_MSFT_opset8_tf_inception_v2_model_cpu"),
-            (xfail_issue_37973, "test_MSFT_opset9_tf_inception_v2_model_cpu"),
-            (xfail_issue_37973, "test_MSFT_opset11_tf_inception_v2_model_cpu"),
-            (xfail_issue_37973, "test_MSFT_opset10_tf_inception_v2_model_cpu"),
+            (xfail_issue_37973, "test_msft_opset7_tf_inception_v2_model_cpu"),
+            (xfail_issue_37973, "test_msft_opset8_tf_inception_v2_model_cpu"),
+            (xfail_issue_37973, "test_msft_opset9_tf_inception_v2_model_cpu"),
+            (xfail_issue_37973, "test_msft_opset11_tf_inception_v2_model_cpu"),
+            (xfail_issue_37973, "test_msft_opset10_tf_inception_v2_model_cpu"),

-            (xfail_issue_58676, "test_MSFT_opset7_fp16_tiny_yolov2_onnxzoo_winmlperf_tiny_yolov2_cpu"),
-            (xfail_issue_58676, "test_MSFT_opset8_fp16_tiny_yolov2_onnxzoo_winmlperf_tiny_yolov2_cpu"),
+            (xfail_issue_58676, "test_msft_opset7_fp16_tiny_yolov2_onnxzoo_winmlperf_tiny_yolov2_cpu"),
+            (xfail_issue_58676, "test_msft_opset8_fp16_tiny_yolov2_onnxzoo_winmlperf_tiny_yolov2_cpu"),

-            (xfail_issue_39669, "test_MSFT_opset9_cgan_cgan_cpu"),
-            (xfail_issue_47495, "test_MSFT_opset10_BERT_Squad_bertsquad10_cpu"),
-            (xfail_issue_78843, "test_MSFT_opset10_mlperf_ssd_mobilenet_300_ssd_mobilenet_v1_coco_2018_01_28_cpu"),
+            (xfail_issue_39669, "test_msft_opset9_cgan_cgan_cpu"),
+            (xfail_issue_47495, "test_msft_opset10_bert_squad_bertsquad10_cpu"),
+            (xfail_issue_78843, "test_msft_opset10_mlperf_ssd_mobilenet_300_ssd_mobilenet_v1_coco_2018_01_28_cpu"),

-            (xfail_issue_86911, "test_MSFT_opset9_LSTM_Seq_lens_unpacked_model_cpu"),
+            (xfail_issue_86911, "test_msft_opset9_lstm_seq_lens_unpacked_model_cpu"),

        ]
        for test_case in import_xfail_list + execution_xfail_list:
--- a/src/frontends/onnx/tests/tests_python/utils/model_importer.py
+++ b/src/frontends/onnx/tests/tests_python/utils/model_importer.py
@ -59,6 +59,8 @@ class ModelImportRunner(onnx.backend.test.BackendTest):
                else:
                    test_name = test_name.replace(string, "_")

+            test_name = test_name.lower()
+
            test_case = ExtOnnxTestCase(
                name=test_name,
                url=None,