openvino/.ci/openvino-onnx/Jenkinsfile

// Copyright (C) 2018-2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0

DOCKER_CONTAINER_NAME= "openvino-onnx-ci-container"
DOCKER_IMAGE_TAG = "openvino-onnx-ci-image"
ONNX_MODEL_ZOO_SHA = "d58213534f2a4d1c4b19ba62b3bb5f544353256e"

BACKEND_CONFIGURATIONS = [
    [ name: "Release", build_type: "Release" ],
    [ name: "Debug", build_type: "Debug" ],
]

// workaround for aborting previous builds on PR update
@NonCPS
 def stopPreviousRunningBuilds() {
     def jobname = env.JOB_NAME
     if (jobname.startsWith("onnx-ci/openvino onnx ci/openvino/PR")){
         def buildnum = env.BUILD_NUMBER.toInteger()
         def job = Jenkins.instance.getItemByFullName(jobname)
         def job_newest = job.builds.first()
         for (build in job.builds.reverse()[0..<-1]) {
             if (build.isBuilding()){
                 echo "Stop task = ${build} because newest #${job_newest} is on the way"
                 build.doStop();
                 continue;
             }
         }
     }

 }

def getGitPrInfo(String project, String workdir) {
    def gitPrInfo = [
        prAuthorEmail : "",
        commitAuthorEmail : "",
        commitHash : "",
        commitSubject : ""
    ]
    try {
        dir ("${workdir}/${project}") {
            gitPrInfo.prAuthorEmail = sh (script: 'git log -1 --pretty="format:%ae" ', returnStdout: true).trim()
            gitPrInfo.commitAuthorEmail = sh (script: 'git log -1 --pretty="format:%ce" ', returnStdout: true).trim()
            gitPrInfo.commitSubject = sh (script: 'git log -1 --pretty="format:%H" ', returnStdout: true).trim()
            gitPrInfo.commitHash = sh (script: 'git log -1 --pretty="format:%s" ', returnStdout: true).trim()
        }
    }
    catch(e) {
        echo "Failed to retrieve ${project} git repository information!"
        echo "ERROR: ${e}"
    }
    return gitPrInfo
}

def notifyByEmail(def gitPrInfo) {
    stage('Notify') {
        String notifyPeople = "${gitPrInfo.prAuthorEmail}, ${gitPrInfo.commitAuthorEmail}"
        emailext (
            subject: "OpenVino CI: PR ${CHANGE_ID} ${currentBuild.result}!",
            body: """
                    Status: ${currentBuild.result}
                    Pull Request Title: ${CHANGE_TITLE}
                    Pull Request: ${CHANGE_URL}
                    Branch: ${CHANGE_BRANCH}
                    Commit Hash: ${gitPrInfo.commitSubject}
                    Commit Subject: ${gitPrInfo.commitHash}
                    Jenkins Build: ${RUN_DISPLAY_URL}
            """,
            to: "${notifyPeople}"
        )
    }
}

def gitSubmoduleUpdate(String repository_name, String workdir) {
    dir ("${workdir}/${repository_name}") {
        sh  label: "Init ${repository_name} submodules",
            script:
        """
            git submodule init && git submodule update \
                --init \
                --no-fetch \
                --recursive
        """
    }
}

def prepare_repository(String workdir) {
    dir("${workdir}") {
        println "Preparing repository in directory: ${workdir}"
        checkout scm
        gitSubmoduleUpdate(PROJECT_NAME, workdir)
    }
}

def updateModels() {
    sh """
        ./src/bindings/python/tests/test_onnx/model_zoo_preprocess.sh -d ${HOME}/ONNX_CI/models_data -o -s ${ONNX_MODEL_ZOO_SHA}
    """
}

def get_docker_container_name(Map configuration){
    println "RUN get_docker_container_name for ${configuration.name}"
    String docker_container_name = "${DOCKER_CONTAINER_NAME}_${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}"
    return docker_container_name
}

def buildDockerImage(Map configuration, String workdir) {
    String docker_image_tag = "${DOCKER_IMAGE_TAG}_${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}".toLowerCase()
    println "docker_image_tag: ${docker_image_tag}"
    updateModels()
    sh """
        docker build --tag=${docker_image_tag} \
        --build-arg BUILD_TYPE=${configuration.build_type} \
        --file=.ci/openvino-onnx/Dockerfile \
        --build-arg http_proxy=${HTTP_PROXY}  \
        --build-arg https_proxy=${HTTPS_PROXY} .
    """
}

def runTests(Map configuration, String workdir) {
    println "Run tests for ${configuration.name}"
    String docker_image_tag = "${DOCKER_IMAGE_TAG}_${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}".toLowerCase()

    String docker_container_name = get_docker_container_name(configuration)

    // Run only basic unit tests in Debug configuration
    if (configuration.build_type == "Debug") {
        sh """
            docker run --name ${docker_container_name} ${docker_image_tag}
        """
    }

    // Run unit-tests AND large model tests by default
    else {
        sh """
            docker run --name ${docker_container_name} \
            --volume ${HOME}/ONNX_CI/models_data/model_zoo/onnx_model_zoo_${ONNX_MODEL_ZOO_SHA}:/root/.onnx/model_zoo/onnx_model_zoo \
            --volume ${HOME}/ONNX_CI/data/model_zoo/MSFT:/root/.onnx/model_zoo/MSFT \
            ${docker_image_tag} /bin/bash -c "tox && tox -e zoo_models"
        """
    }
}

def getConfigurationsMap() {
    def configurationsMap = [:]
    for (backend in BACKEND_CONFIGURATIONS) {
        def configuration = backend.clone()
        configurationsMap[configuration.name] = {
            stage(configuration.name) { CONFIGURATION_WORKFLOW(configuration) }
        }
    }
    return configurationsMap
}

CONFIGURATION_WORKFLOW = { configuration ->
    node("OpenVINO") {
        String workdir = "${HOME}/workspace/${BUILD_NUMBER}_${env.CHANGE_ID}_${configuration.name}"
        try {
            PROJECT_NAME = "openvino"
            stage("Clone repository") {
                    prepare_repository(workdir)
            }
            stage("Prepare Docker environment") {
              dir("${workdir}") {
                    buildDockerImage(configuration, workdir)
                }
            }
            stage("Run tests") {
                timeout(time: 60, unit: 'MINUTES') {
                    runTests(configuration, workdir)
                }
            }
        }
        catch(e) {
                // Set result to ABORTED if exception contains exit code of a process interrupted by SIGTERM
                if ("$e".contains("143")) {
                    currentBuild.result = "ABORTED"
                } else {
                    currentBuild.result = "FAILURE"
                }
                def gitPrInfo = getGitPrInfo(PROJECT_NAME, workdir)
                notifyByEmail(gitPrInfo)
        }
        finally {
            stage("Cleanup") {
                String docker_container_name = get_docker_container_name(configuration)
                sh """
                    docker rm -f ${docker_container_name}
                    rm -rf ${workdir}
                """
            }
        }
    }
}

pipeline {
    agent none
    options {
        skipDefaultCheckout true
        timeout(activity: true, time: 120, unit: 'MINUTES')
    }
    stages {
        stage('Parallel CI') {
            steps {
                stopPreviousRunningBuilds()
                script {
                    parallelStagesMap = getConfigurationsMap()
                    parallel parallelStagesMap
                }
            }

        }
    }
}