Azure CI: Update Linux and Mac (#10163)

* Azure CI: Update Linux and Mac * Update install_build_dependencies.sh
2022-02-07 18:27:06 +03:00
parent d053b6c331
commit d4ad38b78f
6 changed files with 47 additions and 28 deletions
--- a/.ci/azure/linux.yml
+++ b/.ci/azure/linux.yml
@@ -54,6 +54,7 @@ jobs:
      echo Python info ; which python ; python --version
      echo Java info ; which java ; java -version
      echo gcc info ; which gcc ; gcc --version
+      echo cmake info ; which cmake ; cmake --version
      lsb_release
      env
      cat /proc/cpuinfo
@@ -63,14 +64,16 @@ jobs:
      df
      lsblk -o NAME,HCTL,SIZE,MOUNTPOINT | grep -i "sd"
      free -h
+      echo TargetBranch: $(System.PullRequest.TargetBranch)
+      echo SourceBranch: $(Build.SourceBranch)
    displayName: 'System info'

  - script: |
+      set -e
      rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR)
      rm -rf $(BUILD_DIR) ; mkdir $(BUILD_DIR)
      rm -rf $(BUILD_SAMPLES_DIR) ; mkdir $(BUILD_SAMPLES_DIR)
-      echo TargetBranch: $(System.PullRequest.TargetBranch)
-      echo SourceBranch: $(Build.SourceBranch)
+
    displayName: 'Make dir'

  - checkout: self
@@ -91,7 +94,8 @@ jobs:
    path: testdata

  - script: |
-      sudo apt --assume-yes install libusb-1.0-0-dev
+      set -e
+      sudo apt --assume-yes update && sudo apt --assume-yes install libusb-1.0-0-dev
      # For opencv-python: setuptools and upgrade
      sudo apt-get install python3-setuptools patchelf
      python3 -m pip install --upgrade pip
@@ -100,7 +104,7 @@ jobs:
      # For running Python API tests
      python3 -m pip install -r $(REPO_DIR)/inference-engine/ie_bridges/python/src/requirements-dev.txt
      # Speed up build
-      wget https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-linux.zip
+      wget https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-linux.zip
      unzip ninja-linux.zip
      sudo cp -v ninja /usr/local/bin/
      # Speed up tests
--- a/.ci/azure/linux_conditional_compilation.yml
+++ b/.ci/azure/linux_conditional_compilation.yml
@@ -19,14 +19,12 @@ jobs:
    system.debug: true
    VSTS_HTTP_RETRY: 5
    VSTS_HTTP_TIMEOUT: 200
-    WORKERS_NUMBER: 16
    BUILD_TYPE: Release
    REPO_DIR: $(Build.Repository.LocalPath)
    OPENVINO_CONTRIB_REPO_DIR: $(REPO_DIR)/../openvino_contrib
    MODELS_PATH: $(REPO_DIR)/../testdata
    WORK_DIR: $(Pipeline.Workspace)/_w
    BUILD_DIR: $(WORK_DIR)/build
-    BIN_DIR: $(REPO_DIR)/bin/intel64/$(BUILD_TYPE)
    INSTALL_DIR: $(WORK_DIR)/install_pkg
    SETUPVARS: $(INSTALL_DIR)/bin/setupvars.sh

@@ -39,6 +37,7 @@ jobs:
      echo Python info ; which python ; python --version
      echo Java info ; which java ; java -version
      echo gcc info ; which gcc ; gcc --version
+      echo cmake info ; which cmake ; cmake --version
      lsb_release
      env
      cat /proc/cpuinfo
@@ -62,10 +61,11 @@ jobs:
    path: openvino

  - script: |
-      sudo apt --assume-yes install libusb-1.0-0-dev
+      set -e
+      sudo apt --assume-yes update && sudo apt --assume-yes install libusb-1.0-0-dev
      python3 -m pip install -r $(REPO_DIR)/inference-engine/ie_bridges/python/requirements.txt
      # Speed up build
-      wget https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-linux.zip
+      wget https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-linux.zip
      unzip ninja-linux.zip
      sudo cp -v ninja /usr/local/bin/
    workingDirectory: $(WORK_DIR)
@@ -85,12 +85,14 @@ jobs:

  - script: ninja
    workingDirectory: $(BUILD_DIR)
-    displayName: 'Build'
+    displayName: 'Build LinCC'

  - script: ls -alR $(REPO_DIR)/bin/
-    displayName: 'List files'
+    displayName: 'List bin files'

  - script: cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -P cmake_install.cmake
    workingDirectory: $(BUILD_DIR)
    displayName: 'Install'

+  - script: ls -alR $(INSTALL_DIR)
+    displayName: 'List install files'
--- a/.ci/azure/linux_ngraph_onnx.yml
+++ b/.ci/azure/linux_ngraph_onnx.yml
@@ -62,7 +62,7 @@ jobs:
  - script: |
      rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR)
      sudo mkdir -p $(MODELS_DIR)
-      sudo apt --assume-yes install nfs-common
+      sudo apt --assume-yes update && sudo apt --assume-yes install nfs-common
      sudo mount -vvv -t nfs cinfsshare.file.core.windows.net:/cinfsshare/onnxtestdata $(MODELS_DIR) -o vers=4,minorversion=1,sec=sys
      mkdir -p $(MODELS_DIR)/models_data
    displayName: 'Make dirs'
--- a/.ci/azure/linux_onnxruntime.yml
+++ b/.ci/azure/linux_onnxruntime.yml
@@ -12,7 +12,7 @@ jobs:
  timeoutInMinutes: 90

  pool:
-    name: LIN_VMSS_VENV_ONNX_WU2
+    name: LIN_VMSS_VENV_ONNX_U20_WU2

  variables:
    system.debug: true
@@ -28,6 +28,7 @@ jobs:
    BUILD_DIR: $(WORK_DIR)/build
    ONNXRUNTIME_UTILS: $(REPO_DIR)/.ci/azure/ci_utils/onnxruntime
    ONNXRUNTIME_BUILD_DIR: $(ONNXRUNTIME_REPO_DIR)/build
+
  steps:
  - script: |
      curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2019-06-01"
@@ -37,6 +38,7 @@ jobs:
      echo Python info ; which python ; python --version
      echo Java info ; which java ; java -version
      echo gcc info ; which gcc ; gcc --version
+      echo cmake info ; which cmake ; cmake --version
      lsb_release
      env
      cat /proc/cpuinfo
@@ -52,7 +54,7 @@ jobs:
      rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR)
      sudo rm -rf $(TMP_DIR) ; sudo mkdir $(TMP_DIR) ; sudo chmod 777 -R $(TMP_DIR)
      sudo mkdir -p $(MODELS_DIR)
-      sudo apt --assume-yes install nfs-common
+      sudo apt --assume-yes update && sudo apt --assume-yes install nfs-common
      sudo mount -vvv -t nfs cinfsshare.file.core.windows.net:/cinfsshare/onnxtestdata $(MODELS_DIR) -o vers=4,minorversion=1,sec=sys
    displayName: 'Make dirs'

@@ -68,15 +70,14 @@ jobs:
    displayName: 'Clone onnxruntime'

  - script: |
-      sudo apt --assume-yes install libusb-1.0-0-dev
-      # For opencv-python: setuptools and upgrade
-      sudo apt-get install python3-setuptools
+      set -e
+      $(REPO_DIR)/install_build_dependencies.sh
      python3 -m pip install --upgrade pip
      python3 -m pip install -r $(REPO_DIR)/inference-engine/ie_bridges/python/requirements.txt
      # For running Python API tests
      python3 -m pip install -r $(REPO_DIR)/inference-engine/ie_bridges/python/src/requirements-dev.txt
      # Speed up build
-      wget https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-linux.zip
+      wget https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-linux.zip
      unzip ninja-linux.zip
      sudo cp -v ninja /usr/local/bin/
      # Speed up tests
@@ -91,7 +92,7 @@ jobs:
        -GNinja
        -DCMAKE_BUILD_TYPE=$(BUILD_TYPE)
        -DENABLE_PYTHON=ON
-        -DPYTHON_EXECUTABLE=/usr/bin/python3.6
+        -DPYTHON_EXECUTABLE=/usr/bin/python3.8
        -DENABLE_VPU=OFF
        -DENABLE_GNA=OFF
        -DENABLE_OPENCV=OFF
@@ -112,10 +113,10 @@ jobs:

  - script: ninja
    workingDirectory: $(BUILD_DIR)
-    displayName: 'Build Lin'
+    displayName: 'Build Lin ONNX'

  - script: ls -alR $(REPO_DIR)/bin/
-    displayName: 'List files'
+    displayName: 'List bin files'

  - script: cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -P cmake_install.cmake
    workingDirectory: $(BUILD_DIR)
@@ -126,7 +127,7 @@ jobs:
      echo "2021.2" > $(INSTALL_DIR)/deployment_tools/inference_engine/version.txt
      CXXFLAGS="-Wno-error=deprecated-declarations" ./build.sh --config RelWithDebInfo --use_openvino CPU_FP32 --build_shared_lib --parallel --skip_tests --build_dir $(ONNXRUNTIME_BUILD_DIR)
    workingDirectory: $(ONNXRUNTIME_REPO_DIR)
-    displayName: 'Build ONNX Runtime'
+    displayName: 'Build Lin ONNX Runtime'

  - script: |
      source $(INSTALL_DIR)/bin/setupvars.sh
--- a/.ci/azure/mac.yml
+++ b/.ci/azure/mac.yml
@@ -47,11 +47,11 @@ jobs:
  - script: |
      whoami
      uname -a
-      which python3
-      python3 --version
-      which java
-      java -version
-      gcc --version
+      echo Python3 info ; which python3 ; python3 --version
+      echo Python info ; which python ; python --version
+      echo Java info ; which java ; java -version
+      echo gcc info ; which gcc ; gcc --version
+      echo cmake info ; which cmake ; cmake --version
      xcrun --sdk macosx --show-sdk-version
      env
      sysctl -a
@@ -104,17 +104,23 @@ jobs:
    workingDirectory: $(BUILD_DIR)
    displayName: 'CMake'

+  - script: ls -alR $(REPO_DIR)/temp/
+    displayName: 'List temp SDKs'
+
  - script: ninja
    workingDirectory: $(BUILD_DIR)
    displayName: 'Build Mac'

  - script: ls -alR $(REPO_DIR)/bin/
-    displayName: 'List files'
+    displayName: 'List bin files'

  - script: cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -P cmake_install.cmake
    workingDirectory: $(BUILD_DIR)
    displayName: 'Install'

+  - script: ls -alR $(INSTALL_DIR)
+    displayName: 'List install files'
+
  - script: $(BIN_DIR)/unit-test --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU*:IE_CPU.onnx_model_sigmoid:IE_CPU/GRUSequenceOp.onnx_model_gru* --gtest_output=xml:TEST-NGraphUT.xml
    displayName: 'nGraph UT'
    continueOnError: false
--- a/install_build_dependencies.sh
+++ b/install_build_dependencies.sh
@@ -32,6 +32,8 @@ if [ -f /etc/lsb-release ]; then
    sudo -E apt update
    sudo -E apt-get install -y \
            build-essential \
+            cmake \
+            ccache \
            curl \
            wget \
            libssl-dev \
@@ -47,7 +49,11 @@ if [ -f /etc/lsb-release ]; then
            libtool \
            autoconf \
            shellcheck \
-            python \
+            patchelf \
+            libenchant1c2a \
+            python3-pip \
+            python3-enchant \
+            python3-setuptools \
            libcairo2-dev \
            libpango1.0-dev \
            libglib2.0-dev \