IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enable parallel execution open vino onnx ci (#5332)

Browse Source 
* Separate workdir for parallel runs

* Use random

* Get back old working configuration

* Change label

* Debug parameters

* Change models dir to SHA approach

* Fix WORKDIR names

* Change mountig models dirs

* Update model preproces script

* Improvements

* Isolate all private variables

* Fix model dir path

* Update model preproces

* Update node label

* Remove debug printlns
This commit is contained in:
Rafal Blaczkowski 2021-04-22 07:58:11 +02:00 committed by GitHub
parent 9d37403f89
commit 805d184357
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 64 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
.ci/openvino-onnx/Jenkinsfile vendored
View File

@ -3,11 +3,12 @@
DOCKER_CONTAINER_NAME= "openvino-onnx-ci-container" DOCKER_CONTAINER_NAME= "openvino-onnx-ci-container"
DOCKER_IMAGE_TAG = "openvino-onnx-ci-image" DOCKER_IMAGE_TAG = "openvino-onnx-ci-image"
ONNX_MODEL_ZOO_SHA = "d58213534f2a4d1c4b19ba62b3bb5f544353256e"
BACKEND_CONFIGURATIONS = [ BACKEND_CONFIGURATIONS = [
[ name: "Release", build_type: "Release", protobuf_lite : "OFF" ], [ name: "Release", build_type: "Release", protobuf_lite : "OFF" ],
[ name: "Debug", build_type: "Debug", protobuf_lite : "OFF" ], [ name: "Debug", build_type: "Debug", protobuf_lite : "OFF" ],
[ name: "Rel+Lite", build_type: "Release", protobuf_lite : "ON" ], [ name: "Rel_Lite", build_type: "Release", protobuf_lite : "ON" ],
] ]
// workaround for aborting previous builds on PR update // workaround for aborting previous builds on PR update
@ -29,7 +30,7 @@ BACKEND_CONFIGURATIONS = [
} }
def getGitPrInfo(String project) { def getGitPrInfo(String project, String workdir) {
def gitPrInfo = [ def gitPrInfo = [
prAuthorEmail : "", prAuthorEmail : "",
commitAuthorEmail : "", commitAuthorEmail : "",
@ -37,7 +38,7 @@ def getGitPrInfo(String project) {
commitSubject : "" commitSubject : ""
] ]
try { try {
dir ("${WORKDIR}/${project}") { dir ("${workdir}/${project}") {
gitPrInfo.prAuthorEmail = sh (script: 'git log -1 --pretty="format:%ae" ', returnStdout: true).trim() gitPrInfo.prAuthorEmail = sh (script: 'git log -1 --pretty="format:%ae" ', returnStdout: true).trim()
gitPrInfo.commitAuthorEmail = sh (script: 'git log -1 --pretty="format:%ce" ', returnStdout: true).trim() gitPrInfo.commitAuthorEmail = sh (script: 'git log -1 --pretty="format:%ce" ', returnStdout: true).trim()
gitPrInfo.commitSubject = sh (script: 'git log -1 --pretty="format:%H" ', returnStdout: true).trim() gitPrInfo.commitSubject = sh (script: 'git log -1 --pretty="format:%H" ', returnStdout: true).trim()
@ -70,8 +71,8 @@ def notifyByEmail(def gitPrInfo) {
} }
} }
def gitSubmoduleUpdate(String repository_name) { def gitSubmoduleUpdate(String repository_name, String workdir) {
dir ("${WORKDIR}/${repository_name}") { dir ("${workdir}/${repository_name}") {
sh label: "Init ${repository_name} submodules", sh label: "Init ${repository_name} submodules",
script: script:
""" """
@ -83,23 +84,32 @@ def gitSubmoduleUpdate(String repository_name) {
} }
} }
def prepare_repository() { def prepare_repository(String workdir) {
dir("${WORKDIR}") { dir("${workdir}") {
println "Preparing repository in directory: ${workdir}"
checkout scm checkout scm
gitSubmoduleUpdate(PROJECT_NAME) gitSubmoduleUpdate(PROJECT_NAME, workdir)
} }
} }
def updateModels() { def updateModels() {
sh """ sh """
./ngraph/python/tests/test_onnx/model_zoo_preprocess.sh -d ${HOME}/ONNX_CI/data -o ./ngraph/python/tests/test_onnx/model_zoo_preprocess.sh -d ${HOME}/ONNX_CI/models_data -o -s ${ONNX_MODEL_ZOO_SHA}
""" """
} }
def buildDockerImage(Map configuration) { def get_docker_container_name(Map configuration){
println "RUN get_docker_container_name for ${configuration.name}"
String docker_container_name = "${DOCKER_CONTAINER_NAME}_${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}"
return docker_container_name
}
def buildDockerImage(Map configuration, String workdir) {
String docker_image_tag = "${DOCKER_IMAGE_TAG}_${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}".toLowerCase()
println "docker_image_tag: ${docker_image_tag}"
updateModels() updateModels()
sh """ sh """
docker build --tag=${DOCKER_IMAGE_TAG} \ docker build --tag=${docker_image_tag} \
--build-arg BUILD_TYPE=${configuration.build_type} \ --build-arg BUILD_TYPE=${configuration.build_type} \
--build-arg PROTOBUF_LITE=${configuration.protobuf_lite} \ --build-arg PROTOBUF_LITE=${configuration.protobuf_lite} \
--file=.ci/openvino-onnx/Dockerfile \ --file=.ci/openvino-onnx/Dockerfile \
@ -108,20 +118,26 @@ def buildDockerImage(Map configuration) {
""" """
} }
def runTests(Map configuration) { def runTests(Map configuration, String workdir) {
println "Run tests for ${configuration.name}"
String docker_image_tag = "${DOCKER_IMAGE_TAG}_${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}".toLowerCase()
String docker_container_name = get_docker_container_name(configuration)
// Run only basic unit tests in Debug configuration // Run only basic unit tests in Debug configuration
if (configuration.build_type == "Debug") { if (configuration.build_type == "Debug") {
sh """ sh """
docker run --name ${DOCKER_CONTAINER_NAME} ${DOCKER_IMAGE_TAG} docker run --name ${docker_container_name} ${docker_image_tag}
""" """
} }
// Run unit-tests AND large model tests by default // Run unit-tests AND large model tests by default
else { else {
sh """ sh """
docker run --name ${DOCKER_CONTAINER_NAME} \ docker run --name ${docker_container_name} \
--volume ${HOME}/ONNX_CI/data/model_zoo:/root/.onnx/model_zoo \ --volume ${HOME}/ONNX_CI/models_data/model_zoo/onnx_model_zoo_${ONNX_MODEL_ZOO_SHA}:/root/.onnx/model_zoo/onnx_model_zoo \
${DOCKER_IMAGE_TAG} /bin/bash -c "tox && tox -e zoo_models" --volume ${HOME}/ONNX_CI/data/model_zoo/MSFT:/root/.onnx/model_zoo/MSFT \
${docker_image_tag} /bin/bash -c "tox && tox -e zoo_models"
""" """
} }
} }
@ -130,7 +146,6 @@ def getConfigurationsMap() {
def configurationsMap = [:] def configurationsMap = [:]
for (backend in BACKEND_CONFIGURATIONS) { for (backend in BACKEND_CONFIGURATIONS) {
def configuration = backend.clone() def configuration = backend.clone()
configuration.name = "${configuration.name}"
configurationsMap[configuration.name] = { configurationsMap[configuration.name] = {
stage(configuration.name) { CONFIGURATION_WORKFLOW(configuration) } stage(configuration.name) { CONFIGURATION_WORKFLOW(configuration) }
} }
@ -139,29 +154,22 @@ def getConfigurationsMap() {
} }
CONFIGURATION_WORKFLOW = { configuration -> CONFIGURATION_WORKFLOW = { configuration ->
node("OpenVino") { node("OpenVINO") {
try { try {
PROJECT_NAME = "openvino" PROJECT_NAME = "openvino"
WORKDIR = "${HOME}/workspace/${BUILD_NUMBER}" String workdir = "${HOME}/workspace/${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}"
stage("Clone repository") { stage("Clone repository") {
stopPreviousRunningBuilds() prepare_repository(workdir)
try {
prepare_repository()
}
catch(e){
sleep(time: 30, unit: "SECONDS")
prepare_repository()
}
} }
stage("Prepare Docker environment") { stage("Prepare Docker environment") {
dir("${WORKDIR}") { dir("${workdir}") {
buildDockerImage(configuration) buildDockerImage(configuration, workdir)
} }
} }
stage("Run tests") { stage("Run tests") {
timeout(time: 20, unit: 'MINUTES') { timeout(time: 60, unit: 'MINUTES') {
runTests(configuration) runTests(configuration, workdir)
} }
} }
} }
@ -172,15 +180,15 @@ CONFIGURATION_WORKFLOW = { configuration ->
} else { } else {
currentBuild.result = "FAILURE" currentBuild.result = "FAILURE"
} }
gitPrInfo = getGitPrInfo(PROJECT_NAME) def gitPrInfo = getGitPrInfo(PROJECT_NAME, workdir)
notifyByEmail(gitPrInfo) notifyByEmail(gitPrInfo)
} }
finally { finally {
stage("Cleanup") { stage("Cleanup") {
deleteDir() deleteDir()
String docker_container_name = get_docker_container_name(configuration)
sh """ sh """
docker rm -f ${DOCKER_CONTAINER_NAME} docker rm -f ${docker_container_name}
docker image prune -f
""" """
} }
} }
@ -191,11 +199,12 @@ pipeline {
agent none agent none
options { options {
skipDefaultCheckout true skipDefaultCheckout true
timeout(activity: true, time: 60, unit: 'MINUTES') timeout(activity: true, time: 120, unit: 'MINUTES')
} }
stages { stages {
stage('Parallel CI') { stage('Parallel CI') {
steps { steps {
stopPreviousRunningBuilds()
script { script {
parallelStagesMap = getConfigurationsMap() parallelStagesMap = getConfigurationsMap()
parallel parallelStagesMap parallel parallelStagesMap

34
ngraph/python/tests/test_onnx/model_zoo_preprocess.sh
View File

@ -5,7 +5,7 @@
set -e set -e
# provide ONNX Model Zoo commit hash ID to update: # default ONNX Model Zoo commit hash ID:
ONNX_SHA=d58213534f2a4d1c4b19ba62b3bb5f544353256e ONNX_SHA=d58213534f2a4d1c4b19ba62b3bb5f544353256e
MODELS_DIR="$HOME/.onnx/model_zoo" MODELS_DIR="$HOME/.onnx/model_zoo"
@ -20,11 +20,12 @@ function print_help {
echo " -d <DIR> set location of the models (for onnx model ZOO and MSFT models)" echo " -d <DIR> set location of the models (for onnx model ZOO and MSFT models)"
printf " By default the models location is: %s\n" "$HOME/.onnx/model_zoo" printf " By default the models location is: %s\n" "$HOME/.onnx/model_zoo"
echo " -o update Onnx Model Zoo models" echo " -o update Onnx Model Zoo models"
echo " -s Onnx Model Zoo commit SHA"
echo " -m update MSFT models" echo " -m update MSFT models"
echo " -f force update of a chosen model" echo " -f force update of a chosen model"
} }
while getopts ":homfd:" opt; do while getopts "homfd:s:" opt; do
case ${opt} in case ${opt} in
h ) h )
print_help print_help
@ -41,6 +42,9 @@ while getopts ":homfd:" opt; do
o ) o )
ENABLE_ONNX_MODELS_ZOO=true ENABLE_ONNX_MODELS_ZOO=true
;; ;;
s )
ONNX_SHA="$OPTARG"
;;
m ) m )
ENABLE_MSFT_MODELS=true ENABLE_MSFT_MODELS=true
;; ;;
@ -52,7 +56,7 @@ done
shift $((OPTIND -1)) shift $((OPTIND -1))
MODEL_ZOO_DIR="$MODELS_DIR/model_zoo" MODEL_ZOO_DIR="$MODELS_DIR/model_zoo"
ONNX_MODELS_DIR="$MODEL_ZOO_DIR/onnx_model_zoo" ONNX_MODELS_DIR="$MODEL_ZOO_DIR/onnx_model_zoo_$ONNX_SHA"
MSFT_MODELS_DIR="$MODEL_ZOO_DIR/MSFT" MSFT_MODELS_DIR="$MODEL_ZOO_DIR/MSFT"
function pull_and_postprocess_onnx_model_zoo() { function pull_and_postprocess_onnx_model_zoo() {
@ -75,12 +79,23 @@ function pull_and_postprocess_onnx_model_zoo() {
mkdir -p test_data_set_0 mkdir -p test_data_set_0
mv *.pb test_data_set_0/ mv *.pb test_data_set_0/
# Save SHA of successfully post processed repository rm -f $MODEL_ZOO_DIR/executing_$ONNX_SHA
git rev-parse HEAD > "$MODEL_ZOO_DIR/onnx_sha"
} }
function update_onnx_models() { function update_onnx_models() {
if test `find $MODEL_ZOO_DIR/executing_$ONNX_SHA -mmin +60 2>/dev/null`;then
rm -rf $ONNX_MODELS_DIR
rm -f $MODEL_ZOO_DIR/executing_$ONNX_SHA
fi
while [[ -f $MODEL_ZOO_DIR/executing_$ONNX_SHA ]];
do
echo "Onnx Models update are currently executing - sleeping 5 minutes"
sleep 300
done
if [[ ! -d $ONNX_MODELS_DIR ]] ; then if [[ ! -d $ONNX_MODELS_DIR ]] ; then
touch $MODEL_ZOO_DIR/executing_$ONNX_SHA
echo "The ONNX Model Zoo repository doesn't exist on your filesystem then will be cloned" echo "The ONNX Model Zoo repository doesn't exist on your filesystem then will be cloned"
git clone https://github.com/onnx/models.git "$ONNX_MODELS_DIR" git clone https://github.com/onnx/models.git "$ONNX_MODELS_DIR"
cd "$ONNX_MODELS_DIR" cd "$ONNX_MODELS_DIR"
@ -96,15 +111,6 @@ function update_onnx_models() {
git clone https://github.com/onnx/models.git "$ONNX_MODELS_DIR" git clone https://github.com/onnx/models.git "$ONNX_MODELS_DIR"
fi fi
fi fi
cd "$ONNX_MODELS_DIR"
export CURRENT_ONNX_MODELS_SHA=`head -n1 "$MODEL_ZOO_DIR/onnx_sha" 2> /dev/null 2>&1`
if [[ $ONNX_SHA = $CURRENT_ONNX_MODELS_SHA ]] ; then
echo "ONNX Model Zoo repository is in the right state"
else
printf "Current onnx model zoo state is: %s \nChecking out to expected onnx model zoo state: %s \n\n" "$CURRENT_ONNX_MODELS_SHA" "$ONNX_SHA"
pull_and_postprocess_onnx_model_zoo
fi
} }
function update_msft_models() { function update_msft_models() {