From 05c641ff7c0332125e96d1815c69f6692a3c232d Mon Sep 17 00:00:00 2001
From: Rafal Blaczkowski <rafal.blaczkowski@intel.com>
Date: Thu, 16 Sep 2021 13:50:10 +0200
Subject: [PATCH] Update azure onnx linux 2021/4 (#7534)

* update azure pipeline

* save
---
 .ci/azure/linux_ngraph_onnx.yml | 58 ++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 26 deletions(-)

diff --git a/.ci/azure/linux_ngraph_onnx.yml b/.ci/azure/linux_ngraph_onnx.yml
index a5c2a46bd68..5521d224630 100644
--- a/.ci/azure/linux_ngraph_onnx.yml
+++ b/.ci/azure/linux_ngraph_onnx.yml
@@ -1,11 +1,22 @@
 jobs:
-- job: nGraph_ONNX_Lin
+- job: OpenVINO_ONNX_CI
+  strategy:
+    matrix:
+      Release:
+        BUILD_TYPE: 'Release'
+        PROTOBUF_LITE: 'ON'
+        TOX_COMMAND: 'tox && tox -e zoo_models'
+      Debug:
+        BUILD_TYPE: 'Debug'
+        PROTOBUF_LITE: 'ON'
+        TOX_COMMAND: 'tox'
+    maxParallel: 2
 
   # About 300% of total time
   timeoutInMinutes: 90
 
   pool:
-    name: LIN_VMSS_VENV_ONNX_WU2
+    name: LIN_VMSS_VENV_ONNX_U20_WU2
 
   variables:
     system.debug: true
@@ -15,6 +26,8 @@ jobs:
     WORK_DIR: $(Pipeline.Workspace)/_w
     MODELS_DIR: /mount/cinfsshare/onnxtestdata
     TMP_DIR: /mnt/tmp
+    ONNX_MODEL_ZOO_SHA: "d58213534f2a4d1c4b19ba62b3bb5f544353256e"
+
 
   steps:
   - script: |
@@ -25,6 +38,7 @@ jobs:
       echo Python info ; which python ; python --version
       echo Java info ; which java ; java -version
       echo gcc info ; which gcc ; gcc --version
+      echo cmake info ; which cmake ; cmake --version
       lsb_release
       env
       cat /proc/cpuinfo
@@ -38,10 +52,10 @@ jobs:
 
   - script: |
       rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR)
-      sudo rm -rf $(TMP_DIR) ; sudo mkdir $(TMP_DIR) ; sudo chmod 777 -R $(TMP_DIR)
       sudo mkdir -p $(MODELS_DIR)
       sudo apt --assume-yes install nfs-common
       sudo mount -vvv -t nfs cinfsshare.file.core.windows.net:/cinfsshare/onnxtestdata $(MODELS_DIR) -o vers=4,minorversion=1,sec=sys
+      mkdir -p $(MODELS_DIR)/models_data
     displayName: 'Make dirs'
 
   - checkout: self
@@ -50,31 +64,23 @@ jobs:
     submodules: recursive
     path: openvino
 
-  - script: docker build --tag=openvino-onnx-ci-image --file=.ci/openvino-onnx/Dockerfile .
-    displayName: 'Docker build'
-
-  - script: ngraph/python/tests/test_onnx/model_zoo_preprocess.sh -d $(TMP_DIR) -o
-    displayName: 'Get models'
-
   - script: |
-      ##wget -O "$(TMP_DIR)/msft.zip" https://onnxruntimetestdata.blob.core.windows.net/models/20191107.zip
-      ##unzip "$(TMP_DIR)/msft.zip" -d "$(MODELS_DIR)/msft"
-      #unzip "/mnt/onnxtestdata/models/20191107.zip" -d "$(MODELS_DIR)/msft"
-      #mv $(MODELS_DIR)/msft/opset9/LSTM_Seq_lens_unpacked/seq_lens_sorted $(MODELS_DIR)/msft/opset9/LSTM_Seq_lens_unpacked/test_data_set_0
-      #mv $(MODELS_DIR)/msft/opset9/LSTM_Seq_lens_unpacked/seq_lens_unsorted $(MODELS_DIR)/msft/opset9/LSTM_Seq_lens_unpacked/test_data_set_1
-    displayName: 'Get MSFT models'
-    enabled: false
+      set -e
+      sudo apt --assume-yes install git-lfs uidmap
+      curl -fsSL https://get.docker.com -o get-docker.sh
+      sudo sh get-docker.sh
+    workingDirectory: $(WORK_DIR)
+    displayName: 'Install dependencies'
 
-  - script: |
-      ls -alR $(MODELS_DIR)
-      ls -alR $(TMP_DIR)
-    displayName: 'List models'
-    enabled: false
+  - script: ngraph/python/tests/test_onnx/model_zoo_preprocess.sh -d $(MODELS_DIR)/models_data -o -s "$(ONNX_MODEL_ZOO_SHA)"
+    displayName: 'Update models'
+    condition: ne(variables['BUILD_TYPE'], 'Debug')
 
-  - script: sudo fallocate -l 48G /swapfile ; sudo mkswap /swapfile ; sudo swapon /swapfile ; df ; free -h
+  - script: sudo docker build --tag=openvino-onnx-ci-image --file=.ci/openvino-onnx/Dockerfile --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg PROTOBUF_LITE=$(PROTOBUF_LITE) .
+    displayName: 'Docker build $(BUILD_TYPE) protobuf-lite: $(PROTOBUF_LITE)'
+
+  - script: sudo fallocate -l 64G /swapfile ; sudo mkswap /swapfile ; sudo swapon /swapfile ; df ; free -h
     displayName: 'Create swap'
 
-  - script: |
-      docker run --name openvino-onnx-ci-container --volume $(TMP_DIR)/model_zoo:/root/.onnx/model_zoo --volume $(MODELS_DIR)/msft:/root/.onnx/model_zoo/MSFT openvino-onnx-ci-image
-    displayName: 'Docker run'
-
+  - script: sudo docker run --name openvino-onnx-ci-container --volume $(MODELS_DIR)/models_data/model_zoo/onnx_model_zoo_$(ONNX_MODEL_ZOO_SHA):/root/.onnx/model_zoo/onnx_model_zoo --volume $(MODELS_DIR)/msft:/root/.onnx/model_zoo/MSFT openvino-onnx-ci-image /bin/bash -c "$(TOX_COMMAND)"
+    displayName: 'Docker run $(BUILD_TYPE) protobuf-lite: $(PROTOBUF_LITE)'