Enable protobuf-lite in ONNX CI tests (#4487)

.ci/openvino-onnx/Dockerfile

@@ -10,6 +10,7 @@ ENV https_proxy ${https_proxy}
 ENV CI=true
 ENV DEBIAN_FRONTEND=noninteractive
 ENV PYTHONUNBUFFERED 1
+ARG PROTOBUF_LITE=OFF
 
 # Install base dependencies
 RUN apt-get update && apt-get install -y locales && apt-get clean autoclean && apt-get autoremove -y
@@ -68,7 +69,8 @@ RUN cmake .. \
     -DNGRAPH_INTERPRETER_ENABLE=ON \
     -DNGRAPH_DEBUG_ENABLE=OFF \
     -DNGRAPH_DYNAMIC_COMPONENTS_ENABLE=ON \
-    -DCMAKE_INSTALL_PREFIX=/openvino/dist
+    -DCMAKE_INSTALL_PREFIX=/openvino/dist \
+    -DNGRAPH_USE_PROTOBUF_LITE=${PROTOBUF_LITE}
 RUN make -j $(nproc) install
 
 # Run tests via tox

.ci/openvino-onnx/Jenkinsfile

@@ -4,11 +4,16 @@
 DOCKER_CONTAINER_NAME= "openvino-onnx-ci-container"
 DOCKER_IMAGE_TAG = "openvino-onnx-ci-image"
 
+BACKEND_CONFIGURATIONS = [
+    [ protobuf_lite : "ON" ],
+    [ protobuf_lite : "OFF" ]
+]
+
 // workaround for aborting previous builds on PR update
 @NonCPS
  def stopPreviousRunningBuilds() {
      def jobname = env.JOB_NAME
-     if (jobname.startsWith("onnx/openvino_ci/PR")){
+     if (jobname.startsWith("onnx-ci/openvino onnx ci/openvino/PR")){
          def buildnum = env.BUILD_NUMBER.toInteger()
          def job = Jenkins.instance.getItemByFullName(jobname)
          def job_newest = job.builds.first()
@@ -77,16 +82,24 @@ def gitSubmoduleUpdate(String repository_name) {
     }
 }
 
+def prepare_repository() {
+    dir("${WORKDIR}") {
+        checkout scm
+        gitSubmoduleUpdate(PROJECT_NAME)
+    }
+}
+
 def updateModels() {
     sh """
         ./ngraph/python/tests/test_onnx/model_zoo_preprocess.sh -d ${HOME}/ONNX_CI/data -o
     """
 }
 
-def buildDockerImage() {
+def buildDockerImage(String protobuf_lite="OFF") {
     updateModels()
     sh """
-        docker build --tag=${DOCKER_IMAGE_TAG}  --file=.ci/openvino-onnx/Dockerfile \
+        docker build --tag=${DOCKER_IMAGE_TAG} --build-arg PROTOBUF_LITE=${protobuf_lite} \
+        --file=.ci/openvino-onnx/Dockerfile \
         --build-arg http_proxy=http://proxy-chain.intel.com:911/ \
         --build-arg https_proxy=http://proxy-chain.intel.com:912/ .
     """
@@ -100,54 +113,57 @@ def runTests() {
     """
 }
 
+def getConfigurationsMap() {
+    def configurationsMap = [:]
+    for (backend in BACKEND_CONFIGURATIONS) {
+        def configuration = backend.clone()
+        configuration.name = "protobuf-lite ${configuration.protobuf_lite}"
+        configurationsMap[configuration.name] = {
+            stage(configuration.name) { CONFIGURATION_WORKFLOW(configuration) }
+        }
+    }
+    return configurationsMap
+}
 
-pipeline {
-    agent {
-        label "OpenVino"
-    }
-    environment {
+CONFIGURATION_WORKFLOW = { configuration ->
+    node("OpenVino") {
+        try {
             PROJECT_NAME = "openvino"
-        WORKDIR = "${WORKSPACE}/${BUILD_NUMBER}"
-    }
-    options {
-        skipDefaultCheckout true
-        timeout(activity: true, time: 60, unit: 'MINUTES')
-    }
-    stages {
+            WORKDIR = "${HOME}/workspace/${BUILD_NUMBER}"
+
             stage("Clone repository") {
-            steps{
                 stopPreviousRunningBuilds()
-                dir("${WORKDIR}") {
-                    checkout scm
+                try {
+                    prepare_repository()
                 }
-                gitSubmoduleUpdate(PROJECT_NAME)
+                catch(e){
+                    sleep(time: 30, unit: "SECONDS")
+                    prepare_repository()
                 }
             }
             stage("Prepare Docker environment") {
-            steps{
               dir("${WORKDIR}") {
-                    buildDockerImage()
-                }
+                    buildDockerImage(configuration.protobuf_lite)
                 }
             }
             stage("Run tests") {
-            options {
-                  timeout(time: 60, unit: 'MINUTES')
-              }
-            steps{
+                timeout(time: 20, unit: 'MINUTES') {
                     runTests()
                 }
             }
         }
-    post {
-        failure {
-            script {
+        catch(e) {
+                // Set result to ABORTED if exception contains exit code of a process interrupted by SIGTERM
+                if ("$e".contains("143")) {
+                    currentBuild.result = "ABORTED"
+                } else {
+                    currentBuild.result = "FAILURE"
+                }
                 gitPrInfo = getGitPrInfo(PROJECT_NAME)
                 notifyByEmail(gitPrInfo)
         }
-        }
-        cleanup {
-            dir("${WORKDIR}") {
+        finally {
+            stage("Cleanup") {
                 deleteDir()
                 sh """
                     docker image prune -f
@@ -157,3 +173,22 @@ pipeline {
         }
     }
 }
+
+pipeline {
+    agent none
+    options {
+        skipDefaultCheckout true
+        timeout(activity: true, time: 60, unit: 'MINUTES')
+    }
+    stages {
+        stage('Parallel CI') {
+            steps {
+                script {
+                    parallelStagesMap = getConfigurationsMap()
+                    parallel parallelStagesMap
+                }
+            }
+
+        }
+    }
+}

ngraph/python/tests/test_onnx/models/external_data.onnx

@@ -0,0 +1,22 @@
+nGraph ONNX Importer:Á

+&
+data_a
+data_b
+data_cresult"Meantest_mean_example*,
Bdata_cj
+locationdata/tensor.datap
Z
+data_a
+
+

+Z
+data_b
+
+

+Z
+data_c
+
+

+b
+result
+
+

+B

ngraph/python/tests/test_onnx/models/external_data.prototxt

@@ -1,77 +0,0 @@
-ir_version: 3
-producer_name: "nGraph ONNX Importer"
-graph {
-  node {
-    input: "data_a"
-    input: "data_b"
-    input: "data_c"
-    output: "result"
-    op_type: "Mean"
-  }
-  name: "test_mean_example"
-  initializer {
-    dims: 3
-    data_type: 1
-    name: "data_c"
-    external_data {
-        key: "location",
-        value: "data/tensor.data"
-    }
-    data_location: 1
-  }
-  input {
-    name: "data_a"
-    type {
-      tensor_type {
-        elem_type: 1
-        shape {
-          dim {
-            dim_value: 3
-          }
-        }
-      }
-    }
-  }
-  input {
-    name: "data_b"
-    type {
-      tensor_type {
-        elem_type: 1
-        shape {
-          dim {
-            dim_value: 3
-          }
-        }
-      }
-    }
-  }
-  input {
-    name: "data_c"
-    type {
-      tensor_type {
-        elem_type: 1
-        shape {
-          dim {
-            dim_value: 3
-          }
-        }
-      }
-    }
-  }
-  output {
-    name: "result"
-    type {
-      tensor_type {
-        elem_type: 1
-        shape {
-          dim {
-            dim_value: 3
-          }
-        }
-      }
-    }
-  }
-}
-opset_import {
-  version: 8
-}

ngraph/python/tests/test_onnx/test_onnx_external_data.py

@@ -24,7 +24,7 @@ from tests.runtime import get_runtime
 
 
 def test_import_onnx_with_external_data():
-    model_path = os.path.join(os.path.dirname(__file__), "models/external_data.prototxt")
+    model_path = os.path.join(os.path.dirname(__file__), "models/external_data.onnx")
     ie = IECore()
     ie_network = ie.read_network(model=model_path)