From 783af1e5ae7b3d36f2eef8565c465c65b3915773 Mon Sep 17 00:00:00 2001 From: azhogov Date: Wed, 10 Feb 2021 17:34:19 +0300 Subject: [PATCH] Azure CI: Sync with master --- .ci/azure/linux.yml | 14 ++-- .ci/azure/linux_ngraph_onnx.yml | 125 ++++++++++++++------------------ .ci/azure/windows.yml | 10 +-- 3 files changed, 66 insertions(+), 83 deletions(-) diff --git a/.ci/azure/linux.yml b/.ci/azure/linux.yml index 0a509161403..ad75561759f 100644 --- a/.ci/azure/linux.yml +++ b/.ci/azure/linux.yml @@ -4,13 +4,13 @@ jobs: timeoutInMinutes: 90 pool: - name: LIN_VMSS_VENV_F8S_WU2 + name: LIN_VMSS_VENV_F16S_WU2 variables: system.debug: true VSTS_HTTP_RETRY: 5 VSTS_HTTP_TIMEOUT: 200 - WORKERS_NUMBER: 8 + WORKERS_NUMBER: 16 BUILD_TYPE: Release REPO_DIR: $(Build.Repository.LocalPath) WORK_DIR: $(Pipeline.Workspace)/_w @@ -22,11 +22,10 @@ jobs: curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2019-06-01" whoami uname -a - which python3 - python3 --version - which java - java -version - gcc --version + echo Python3 info ; which python3 ; python3 --version + echo Python info ; which python ; python --version + echo Java info ; which java ; java -version + echo gcc info ; which gcc ; gcc --version lsb_release env cat /proc/cpuinfo @@ -35,6 +34,7 @@ jobs: vmstat -s df lsblk -o NAME,HCTL,SIZE,MOUNTPOINT | grep -i "sd" + free -h displayName: 'System info' - script: | diff --git a/.ci/azure/linux_ngraph_onnx.yml b/.ci/azure/linux_ngraph_onnx.yml index f993670f98c..c2ee9f1c31b 100644 --- a/.ci/azure/linux_ngraph_onnx.yml +++ b/.ci/azure/linux_ngraph_onnx.yml @@ -1,11 +1,11 @@ jobs: - job: nGraph_ONNX_Lin - # About 150% of total time - timeoutInMinutes: 60 + # About 300% of total time + timeoutInMinutes: 90 pool: - name: LIN_VMSS_VENV_F8S_WU2 + name: LIN_VMSS_VENV_ONNX_WU2 variables: system.debug: true @@ -15,81 +15,68 @@ jobs: BUILD_TYPE: Release REPO_DIR: $(Build.Repository.LocalPath) WORK_DIR: $(Pipeline.Workspace)/_w - BUILD_DIR: $(WORK_DIR)/build - BIN_DIR: $(REPO_DIR)/bin/intel64/$(BUILD_TYPE) - INSTALL_DIR: $(WORK_DIR)/install + MODELS_DIR: /mount/cinfsshare/onnxtestdata + TMP_DIR: /mnt/tmp steps: + - script: | + curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2019-06-01" + whoami + uname -a + echo Python3 info ; which python3 ; python3 --version + echo Python info ; which python ; python --version + echo Java info ; which java ; java -version + echo gcc info ; which gcc ; gcc --version + lsb_release + env + cat /proc/cpuinfo + cat /proc/meminfo + cat /etc/fstab + vmstat -s + df + lsblk -o NAME,HCTL,SIZE,MOUNTPOINT | grep -i "sd" + free -h + displayName: 'System info' + + - script: | + rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR) + sudo rm -rf $(TMP_DIR) ; sudo mkdir $(TMP_DIR) ; sudo chmod 777 -R $(TMP_DIR) + sudo mkdir -p $(MODELS_DIR) + sudo apt --assume-yes install nfs-common + sudo mount -t nfs cinfsshare.file.core.windows.net:/cinfsshare/onnxtestdata $(MODELS_DIR) -o vers=4,minorversion=1,sec=sys + displayName: 'Make dirs' + - checkout: self clean: true lfs: false submodules: recursive path: openvino - - script: | - curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2019-06-01" - whoami - uname -a - which python3 - python3 --version - gcc --version - lsb_release - env - cat /proc/cpuinfo - cat /proc/meminfo - vmstat -s - df - displayName: 'System info' - - - script: | - rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR) - displayName: 'Make dir' - - - script: | - sudo apt --assume-yes install libusb-1.0-0-dev - python3 -m pip install -r ./inference-engine/ie_bridges/python/requirements.txt - # For running Python API tests - python3 -m pip install -r ./inference-engine/ie_bridges/python/src/requirements-dev.txt - displayName: 'Install dependencies' - enabled: false - - - script: | - wget https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-linux.zip - unzip ninja-linux.zip - sudo cp -v ninja /usr/local/bin/ - workingDirectory: $(WORK_DIR) - displayName: 'Install Ninja' - enabled: false - - - task: CMake@1 - inputs: - # CMake must get Python 3.x version by default - cmakeArgs: -GNinja -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_VPU=OFF -DENABLE_GNA=OFF -DENABLE_OPENCV=OFF -DENABLE_CPPLINT=OFF -DENABLE_TESTS=OFF -DENABLE_BEH_TESTS=OFF -DENABLE_FUNCTIONAL_TESTS=OFF -DENABLE_MKL_DNN=ON -DENABLE_CLDNN=OFF -DENABLE_PROFILING_ITT=OFF -DENABLE_SAMPLES=OFF -DENABLE_SPEECH_DEMO=OFF -DENABLE_PYTHON=ON -DPYTHON_EXECUTABLE=/usr/bin/python3.6 -DNGRAPH_ONNX_IMPORT_ENABLE=ON -DNGRAPH_INTERPRETER_ENABLE=ON -DNGRAPH_DEBUG_ENABLE=OFF -DNGRAPH_DYNAMIC_COMPONENTS_ENABLE=ON -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) $(REPO_DIR) - workingDirectory: $(BUILD_DIR) - enabled: false - - - script: ninja - workingDirectory: $(BUILD_DIR) - displayName: 'Build' - enabled: false - - - script: make install - workingDirectory: $(BUILD_DIR) - displayName: 'Install' - enabled: false - - - script: | - ls -alR $(REPO_DIR)/bin/ - ls -alR $(INSTALL_DIR) - displayName: 'List files' - enabled: false - - - script: docker build --tag=openvino-onnx-ci-image --file=$(REPO_DIR)/.ci/openvino-onnx/Dockerfile . - workingDirectory: $(BUILD_DIR) + - script: docker build --tag=openvino-onnx-ci-image --file=.ci/openvino-onnx/Dockerfile . displayName: 'Docker build' + + - script: ngraph/python/tests/test_onnx/model_zoo_preprocess.sh -d $(TMP_DIR) -o + displayName: 'Get models' + + - script: | + ##wget -O "$(TMP_DIR)/msft.zip" https://onnxruntimetestdata.blob.core.windows.net/models/20191107.zip + ##unzip "$(TMP_DIR)/msft.zip" -d "$(MODELS_DIR)/msft" + #unzip "/mnt/onnxtestdata/models/20191107.zip" -d "$(MODELS_DIR)/msft" + #mv $(MODELS_DIR)/msft/opset9/LSTM_Seq_lens_unpacked/seq_lens_sorted $(MODELS_DIR)/msft/opset9/LSTM_Seq_lens_unpacked/test_data_set_0 + #mv $(MODELS_DIR)/msft/opset9/LSTM_Seq_lens_unpacked/seq_lens_unsorted $(MODELS_DIR)/msft/opset9/LSTM_Seq_lens_unpacked/test_data_set_1 + displayName: 'Get MSFT models' enabled: false - - script: docker run --name openvino-onnx-ci-container openvino-onnx-ci-image - workingDirectory: $(BUILD_DIR) - displayName: 'Docker run tests' + - script: | + ls -alR $(MODELS_DIR) + ls -alR $(TMP_DIR) + displayName: 'List models' enabled: false + + - script: sudo fallocate -l 48G /swapfile ; sudo mkswap /swapfile ; sudo swapon /swapfile ; df ; free -h + displayName: 'Create swap' + + - script: | + docker run --name openvino-onnx-ci-container --volume $(TMP_DIR)/model_zoo:/root/.onnx/model_zoo --volume $(MODELS_DIR)/msft:/root/.onnx/model_zoo/MSFT openvino-onnx-ci-image + displayName: 'Docker run' + diff --git a/.ci/azure/windows.yml b/.ci/azure/windows.yml index 90073808dca..cb451dd25c3 100644 --- a/.ci/azure/windows.yml +++ b/.ci/azure/windows.yml @@ -53,16 +53,11 @@ jobs: displayName: 'Install dependencies' - script: | - certutil -urlcache -split -f https://incredibuilddiag1wu2.blob.core.windows.net/incredibuild/IBSetupConsole_9_5_0.exe IBSetupConsole_9_5_0.exe - call IBSetupConsole_9_5_0.exe /Install /Components=Agent,oneuse /Coordinator=11.1.0.4 /AGENT:OPENFIREWALL=ON /AGENT:AUTOSELECTPORTS=ON /ADDTOPATH=ON /AGENT:INSTALLADDINS=OFF + certutil -urlcache -split -f https://incredibuilddiag1wu2.blob.core.windows.net/incredibuild/install_ib_console.bat install_ib_console.bat + call install_ib_console.bat workingDirectory: $(WORK_DIR) displayName: 'Install IncrediBuild' - - script: | - echo Stop IncrediBuild_Agent && net stop IncrediBuild_Agent - reg add HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Xoreax\IncrediBuild\Builder /f /v LastEnabled /d 0 && echo Start IncrediBuild_Agent && net start IncrediBuild_Agent - displayName: 'Start IncrediBuild' - - script: | set PATH=$(WORK_DIR)\ninja-win;%PATH% call "$(MSVS_VARS_PATH)" && cmake -GNinja -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_TESTS=ON -DCMAKE_C_COMPILER:PATH="$(MSVC_COMPILER_PATH)" -DCMAKE_CXX_COMPILER:PATH="$(MSVC_COMPILER_PATH)" $(REPO_DIR) @@ -78,6 +73,7 @@ jobs: - script: echo Stop IncrediBuild_Agent && net stop IncrediBuild_Agent displayName: Stop IncrediBuild continueOnError: true + - script: dir $(REPO_DIR)\bin\ /s displayName: 'List files'