IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

OpenVINO ONNX CI Azure - update onnx/models on demand only (#7125)

Browse Source 
* debug

* debug2

* debug3

* save

* save

* save

* fix

* save

* workaround

* fix2

* remove old artifacts

* final version
This commit is contained in:
Rafal Blaczkowski 2021-08-18 12:57:30 +02:00 committed by GitHub
parent 42e1a49e3a
commit 1247b228db
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 7 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.ci/azure/linux_ngraph_onnx.yml
View File

@ -52,10 +52,10 @@ jobs:
- script: | - script: |
rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR) rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR)
sudo rm -rf $(TMP_DIR) ; sudo mkdir $(TMP_DIR) ; sudo chmod 777 -R $(TMP_DIR)
sudo mkdir -p $(MODELS_DIR) sudo mkdir -p $(MODELS_DIR)
sudo apt --assume-yes install nfs-common sudo apt --assume-yes install nfs-common
sudo mount -vvv -t nfs cinfsshare.file.core.windows.net:/cinfsshare/onnxtestdata $(MODELS_DIR) -o vers=4,minorversion=1,sec=sys sudo mount -vvv -t nfs cinfsshare.file.core.windows.net:/cinfsshare/onnxtestdata $(MODELS_DIR) -o vers=4,minorversion=1,sec=sys
mkdir -p $(MODELS_DIR)/models_data
displayName: 'Make dirs' displayName: 'Make dirs'
- checkout: self - checkout: self
@ -72,15 +72,15 @@ jobs:
workingDirectory: $(WORK_DIR) workingDirectory: $(WORK_DIR)
displayName: 'Install dependencies' displayName: 'Install dependencies'
- script: ngraph/python/tests/test_onnx/model_zoo_preprocess.sh -d $(MODELS_DIR)/models_data -o -s "$(ONNX_MODEL_ZOO_SHA)"
displayName: 'Update models'
condition: ne(variables['BUILD_TYPE'], 'Debug')
- script: sudo docker build --tag=openvino-onnx-ci-image --file=.ci/openvino-onnx/Dockerfile --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg PROTOBUF_LITE=$(PROTOBUF_LITE) . - script: sudo docker build --tag=openvino-onnx-ci-image --file=.ci/openvino-onnx/Dockerfile --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg PROTOBUF_LITE=$(PROTOBUF_LITE) .
displayName: 'Docker build $(BUILD_TYPE) protobuf-lite: $(PROTOBUF_LITE)' displayName: 'Docker build $(BUILD_TYPE) protobuf-lite: $(PROTOBUF_LITE)'
- script: ngraph/python/tests/test_onnx/model_zoo_preprocess.sh -d $(TMP_DIR) -o -s "$(ONNX_MODEL_ZOO_SHA)"
displayName: 'Get models'
condition: ne(variables['BUILD_TYPE'], 'Debug')
- script: sudo fallocate -l 64G /swapfile ; sudo mkswap /swapfile ; sudo swapon /swapfile ; df ; free -h - script: sudo fallocate -l 64G /swapfile ; sudo mkswap /swapfile ; sudo swapon /swapfile ; df ; free -h
displayName: 'Create swap' displayName: 'Create swap'
- script: sudo docker run --name openvino-onnx-ci-container --volume $(TMP_DIR)/model_zoo/onnx_model_zoo_$(ONNX_MODEL_ZOO_SHA):/root/.onnx/model_zoo/onnx_model_zoo --volume $(MODELS_DIR)/msft:/root/.onnx/model_zoo/MSFT openvino-onnx-ci-image /bin/bash -c "$(TOX_COMMAND)" - script: sudo docker run --name openvino-onnx-ci-container --volume $(MODELS_DIR)/models_data/model_zoo/onnx_model_zoo_$(ONNX_MODEL_ZOO_SHA):/root/.onnx/model_zoo/onnx_model_zoo --volume $(MODELS_DIR)/msft:/root/.onnx/model_zoo/MSFT openvino-onnx-ci-image /bin/bash -c "$(TOX_COMMAND)"
displayName: 'Docker run $(BUILD_TYPE) protobuf-lite: $(PROTOBUF_LITE)' displayName: 'Docker run $(BUILD_TYPE) protobuf-lite: $(PROTOBUF_LITE)'

1
ngraph/python/tests/test_onnx/model_zoo_preprocess.sh
View File

@ -96,6 +96,7 @@ function update_onnx_models() {
if [[ ! -d $ONNX_MODELS_DIR ]] ; then if [[ ! -d $ONNX_MODELS_DIR ]] ; then
touch $MODEL_ZOO_DIR/executing_$ONNX_SHA touch $MODEL_ZOO_DIR/executing_$ONNX_SHA
trap "rm -f $MODEL_ZOO_DIR/executing_$ONNX_SHA" EXIT INT TERM
echo "The ONNX Model Zoo repository doesn't exist on your filesystem then will be cloned" echo "The ONNX Model Zoo repository doesn't exist on your filesystem then will be cloned"
git clone https://github.com/onnx/models.git "$ONNX_MODELS_DIR" git clone https://github.com/onnx/models.git "$ONNX_MODELS_DIR"
cd "$ONNX_MODELS_DIR" cd "$ONNX_MODELS_DIR"