Azure CI: Sync with master

2021-02-10 17:34:19 +03:00
parent 8396e6473e
commit 783af1e5ae
3 changed files with 66 additions and 83 deletions
--- a/.ci/azure/linux.yml
+++ b/.ci/azure/linux.yml
@@ -4,13 +4,13 @@ jobs:
  timeoutInMinutes: 90

  pool:
-    name: LIN_VMSS_VENV_F8S_WU2
+    name: LIN_VMSS_VENV_F16S_WU2

  variables:
    system.debug: true
    VSTS_HTTP_RETRY: 5
    VSTS_HTTP_TIMEOUT: 200
-    WORKERS_NUMBER: 8
+    WORKERS_NUMBER: 16
    BUILD_TYPE: Release
    REPO_DIR: $(Build.Repository.LocalPath)
    WORK_DIR: $(Pipeline.Workspace)/_w
@@ -22,11 +22,10 @@ jobs:
      curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2019-06-01"
      whoami
      uname -a
-      which python3
-      python3 --version
-      which java
-      java -version
-      gcc --version
+      echo Python3 info ; which python3 ; python3 --version
+      echo Python info ; which python ; python --version
+      echo Java info ; which java ; java -version
+      echo gcc info ; which gcc ; gcc --version
      lsb_release
      env
      cat /proc/cpuinfo
@@ -35,6 +34,7 @@ jobs:
      vmstat -s
      df
      lsblk -o NAME,HCTL,SIZE,MOUNTPOINT | grep -i "sd"
+      free -h
    displayName: 'System info'

  - script: |
--- a/.ci/azure/linux_ngraph_onnx.yml
+++ b/.ci/azure/linux_ngraph_onnx.yml
@@ -1,11 +1,11 @@
 jobs:
 - job: nGraph_ONNX_Lin

-  # About 150% of total time
-  timeoutInMinutes: 60
+  # About 300% of total time
+  timeoutInMinutes: 90

  pool:
-    name: LIN_VMSS_VENV_F8S_WU2
+    name: LIN_VMSS_VENV_ONNX_WU2

  variables:
    system.debug: true
@@ -15,81 +15,68 @@ jobs:
    BUILD_TYPE: Release
    REPO_DIR: $(Build.Repository.LocalPath)
    WORK_DIR: $(Pipeline.Workspace)/_w
-    BUILD_DIR: $(WORK_DIR)/build
-    BIN_DIR: $(REPO_DIR)/bin/intel64/$(BUILD_TYPE)
-    INSTALL_DIR: $(WORK_DIR)/install
+    MODELS_DIR: /mount/cinfsshare/onnxtestdata
+    TMP_DIR: /mnt/tmp

  steps:
+  - script: |
+      curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2019-06-01"
+      whoami
+      uname -a
+      echo Python3 info ; which python3 ; python3 --version
+      echo Python info ; which python ; python --version
+      echo Java info ; which java ; java -version
+      echo gcc info ; which gcc ; gcc --version
+      lsb_release
+      env
+      cat /proc/cpuinfo
+      cat /proc/meminfo
+      cat /etc/fstab
+      vmstat -s
+      df
+      lsblk -o NAME,HCTL,SIZE,MOUNTPOINT | grep -i "sd"
+      free -h
+    displayName: 'System info'
+
+  - script: |
+      rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR)
+      sudo rm -rf $(TMP_DIR) ; sudo mkdir $(TMP_DIR) ; sudo chmod 777 -R $(TMP_DIR)
+      sudo mkdir -p $(MODELS_DIR)
+      sudo apt --assume-yes install nfs-common
+      sudo mount -t nfs cinfsshare.file.core.windows.net:/cinfsshare/onnxtestdata $(MODELS_DIR) -o vers=4,minorversion=1,sec=sys
+    displayName: 'Make dirs'
+
  - checkout: self
    clean: true
    lfs: false
    submodules: recursive
    path: openvino

-  - script: |
-      curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2019-06-01"
-      whoami
-      uname -a
-      which python3
-      python3 --version
-      gcc --version
-      lsb_release
-      env
-      cat /proc/cpuinfo
-      cat /proc/meminfo
-      vmstat -s
-      df
-    displayName: 'System info'
-
-  - script: |
-      rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR)
-    displayName: 'Make dir'
-
-  - script: |
-      sudo apt --assume-yes install libusb-1.0-0-dev
-      python3 -m pip install -r ./inference-engine/ie_bridges/python/requirements.txt
-      # For running Python API tests
-      python3 -m pip install -r ./inference-engine/ie_bridges/python/src/requirements-dev.txt
-    displayName: 'Install dependencies'
-    enabled: false
-
-  - script: |
-      wget https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-linux.zip
-      unzip ninja-linux.zip
-      sudo cp -v ninja /usr/local/bin/
-    workingDirectory: $(WORK_DIR)
-    displayName: 'Install Ninja'
-    enabled: false
-
-  - task: CMake@1
-    inputs:
-      # CMake must get Python 3.x version by default
-      cmakeArgs: -GNinja -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_VPU=OFF -DENABLE_GNA=OFF -DENABLE_OPENCV=OFF -DENABLE_CPPLINT=OFF -DENABLE_TESTS=OFF -DENABLE_BEH_TESTS=OFF -DENABLE_FUNCTIONAL_TESTS=OFF -DENABLE_MKL_DNN=ON -DENABLE_CLDNN=OFF -DENABLE_PROFILING_ITT=OFF -DENABLE_SAMPLES=OFF -DENABLE_SPEECH_DEMO=OFF -DENABLE_PYTHON=ON -DPYTHON_EXECUTABLE=/usr/bin/python3.6 -DNGRAPH_ONNX_IMPORT_ENABLE=ON -DNGRAPH_INTERPRETER_ENABLE=ON -DNGRAPH_DEBUG_ENABLE=OFF -DNGRAPH_DYNAMIC_COMPONENTS_ENABLE=ON -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) $(REPO_DIR)
-      workingDirectory: $(BUILD_DIR)
-    enabled: false
-
-  - script: ninja
-    workingDirectory: $(BUILD_DIR)
-    displayName: 'Build'
-    enabled: false
-
-  - script: make install
-    workingDirectory: $(BUILD_DIR)
-    displayName: 'Install'
-    enabled: false
-
-  - script: |
-      ls -alR $(REPO_DIR)/bin/
-      ls -alR $(INSTALL_DIR)
-    displayName: 'List files'
-    enabled: false
-
-  - script: docker build --tag=openvino-onnx-ci-image --file=$(REPO_DIR)/.ci/openvino-onnx/Dockerfile .
-    workingDirectory: $(BUILD_DIR)
+  - script: docker build --tag=openvino-onnx-ci-image --file=.ci/openvino-onnx/Dockerfile .
    displayName: 'Docker build'
+
+  - script: ngraph/python/tests/test_onnx/model_zoo_preprocess.sh -d $(TMP_DIR) -o
+    displayName: 'Get models'
+
+  - script: |
+      ##wget -O "$(TMP_DIR)/msft.zip" https://onnxruntimetestdata.blob.core.windows.net/models/20191107.zip
+      ##unzip "$(TMP_DIR)/msft.zip" -d "$(MODELS_DIR)/msft"
+      #unzip "/mnt/onnxtestdata/models/20191107.zip" -d "$(MODELS_DIR)/msft"
+      #mv $(MODELS_DIR)/msft/opset9/LSTM_Seq_lens_unpacked/seq_lens_sorted $(MODELS_DIR)/msft/opset9/LSTM_Seq_lens_unpacked/test_data_set_0
+      #mv $(MODELS_DIR)/msft/opset9/LSTM_Seq_lens_unpacked/seq_lens_unsorted $(MODELS_DIR)/msft/opset9/LSTM_Seq_lens_unpacked/test_data_set_1
+    displayName: 'Get MSFT models'
    enabled: false

-  - script: docker run --name openvino-onnx-ci-container openvino-onnx-ci-image
-    workingDirectory: $(BUILD_DIR)
-    displayName: 'Docker run tests'
+  - script: |
+      ls -alR $(MODELS_DIR)
+      ls -alR $(TMP_DIR)
+    displayName: 'List models'
    enabled: false
+
+  - script: sudo fallocate -l 48G /swapfile ; sudo mkswap /swapfile ; sudo swapon /swapfile ; df ; free -h
+    displayName: 'Create swap'
+
+  - script: |
+      docker run --name openvino-onnx-ci-container --volume $(TMP_DIR)/model_zoo:/root/.onnx/model_zoo --volume $(MODELS_DIR)/msft:/root/.onnx/model_zoo/MSFT openvino-onnx-ci-image
+    displayName: 'Docker run'
+
--- a/.ci/azure/windows.yml
+++ b/.ci/azure/windows.yml
@@ -53,16 +53,11 @@ jobs:
    displayName: 'Install dependencies'

  - script: |
-      certutil -urlcache -split -f https://incredibuilddiag1wu2.blob.core.windows.net/incredibuild/IBSetupConsole_9_5_0.exe IBSetupConsole_9_5_0.exe
-      call IBSetupConsole_9_5_0.exe /Install /Components=Agent,oneuse /Coordinator=11.1.0.4 /AGENT:OPENFIREWALL=ON /AGENT:AUTOSELECTPORTS=ON /ADDTOPATH=ON /AGENT:INSTALLADDINS=OFF
+      certutil -urlcache -split -f https://incredibuilddiag1wu2.blob.core.windows.net/incredibuild/install_ib_console.bat install_ib_console.bat
+      call install_ib_console.bat
    workingDirectory: $(WORK_DIR)
    displayName: 'Install IncrediBuild'

-  - script: |
-      echo Stop IncrediBuild_Agent && net stop IncrediBuild_Agent
-      reg add HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Xoreax\IncrediBuild\Builder /f /v LastEnabled /d 0 && echo Start IncrediBuild_Agent && net start IncrediBuild_Agent
-    displayName: 'Start IncrediBuild'
-
  - script: |
      set PATH=$(WORK_DIR)\ninja-win;%PATH%
      call "$(MSVS_VARS_PATH)" && cmake -GNinja -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_TESTS=ON -DCMAKE_C_COMPILER:PATH="$(MSVC_COMPILER_PATH)" -DCMAKE_CXX_COMPILER:PATH="$(MSVC_COMPILER_PATH)" $(REPO_DIR)
@@ -78,6 +73,7 @@ jobs:
  - script: echo Stop IncrediBuild_Agent && net stop IncrediBuild_Agent
    displayName: Stop IncrediBuild
    continueOnError: true
+
  - script: dir $(REPO_DIR)\bin\ /s
    displayName: 'List files'