From 80f60b9cfa75958ab7628c7e4320e56459c26fda Mon Sep 17 00:00:00 2001 From: Arne Morten Kvarving Date: Thu, 26 May 2016 17:15:09 +0200 Subject: [PATCH] added: jenkins build scripts with downstream building support --- Apps/Common/scripts/regtest.sh.in | 2 +- cmake/Scripts/IFEMTesting.cmake | 2 +- jenkins/README.md | 30 ++++ jenkins/build-ifem-module.sh | 252 ++++++++++++++++++++++++++++++ jenkins/build.sh | 49 ++++++ jenkins/conv.xsl | 116 ++++++++++++++ jenkins/convert.py | 74 +++++++++ 7 files changed, 523 insertions(+), 2 deletions(-) create mode 100644 jenkins/README.md create mode 100755 jenkins/build-ifem-module.sh create mode 100755 jenkins/build.sh create mode 100644 jenkins/conv.xsl create mode 100755 jenkins/convert.py diff --git a/Apps/Common/scripts/regtest.sh.in b/Apps/Common/scripts/regtest.sh.in index 631cd96c..3d4937b2 100755 --- a/Apps/Common/scripts/regtest.sh.in +++ b/Apps/Common/scripts/regtest.sh.in @@ -17,7 +17,7 @@ mysim=$1 cd `dirname $2` MAPFILE=`head -n1 $2` test $? -eq 0 || exit 1 -test -n "$3" && mysim="$PETSC_DIR/bin/petscmpiexec -n $3 $mysim" +test -n "$3" && mysim="mpirun -n $3 $mysim" $mysim $MAPFILE 2>&1 > @CMAKE_BINARY_DIR@/templog appres=$? globres=1 diff --git a/cmake/Scripts/IFEMTesting.cmake b/cmake/Scripts/IFEMTesting.cmake index 34970670..db581580 100644 --- a/cmake/Scripts/IFEMTesting.cmake +++ b/cmake/Scripts/IFEMTesting.cmake @@ -41,7 +41,7 @@ macro(IFEM_add_test_app path workdir name) add_executable(${name}-test EXCLUDE_FROM_ALL ${IFEM_PATH}/src/IFEM-test.C ${TEST_SRCS}) gtest_add_tests($ ${workdir} ${TEST_SRCS}) list(APPEND TEST_APPS ${name}-test) - target_link_libraries(${name}-test ${ARGN} gtest) + target_link_libraries(${name}-test ${ARGN} gtest pthread) endmacro() macro(IFEM_add_unittests IFEM_PATH) diff --git a/jenkins/README.md b/jenkins/README.md new file mode 100644 index 00000000..c0e3af70 --- /dev/null +++ b/jenkins/README.md @@ -0,0 +1,30 @@ +# IFEM jenkins build scripts: + +**build-ifem-module.sh**: +This is a helper script which contains functions for building, +testing and cloning modules. + +**build.sh**: +This expects to run on a jenkins instance with IFEM as the 'origin' remote. + +It will build and test IFEM. It can be used both for post-merge builds +of the master branch and for a github pull request builder job. +Optionally you it can build all downstreams and execute their tests. + +Important environment variables (jenkins conventions): + +WORKSPACE - Root of source tree. + +ghprbCommentBody - Trigger as used on github. + +GH\_CREDENTIALS - Github credentials to use in the form user:password. + +CMAKE\_TOOLCHAIN\_FILES - A space separated list of toolchain files specifying + cmake parameters for a configuration. + +BTYPES - A space separated list of build configuration names. + +A typically local run building the current content of IFEM and all +downstreams; + +WORKSPACE=`pwd` GH\_CREDENTIALS=user:pwd ghprbCommentBody="jenkins build this with downstreams please" jenkins/build.sh diff --git a/jenkins/build-ifem-module.sh b/jenkins/build-ifem-module.sh new file mode 100755 index 00000000..98485b4c --- /dev/null +++ b/jenkins/build-ifem-module.sh @@ -0,0 +1,252 @@ +#!/bin/bash + +# Hacks due to not following standards +declare -A MODULE_EXTRA_DIR +MODULE_EXTRA_DIR[IFEM-BeamEx]=IFEM-Elasticity/ +MODULE_EXTRA_DIR[IFEM-FiniteDeformation]=IFEM-Elasticity/ + +declare -A MODULE_APP_DIR +MODULE_APP_DIR[IFEM-BeamEx]=BeamSim +MODULE_APP_DIR[IFEM-Elasticity]=Linear +MODULE_APP_DIR[IFEM-FiniteDeformation]=Nonlinear + + +# Parse revisions from trigger comment and setup arrays +function parseRevisions { + for upstream in ${upstreams[*]} + do + if grep -qi "$upstream=" <<< $ghprbCommentBody + then + upstreamRev[$upstream]=pull/`echo $ghprbCommentBody | sed -r "s/.*${upstream,,}=([0-9]+).*/\1/g"`/merge + fi + done + if grep -q "with downstreams" <<< $ghprbCommentBody + then + for downstream in ${downstreams[*]} + do + if grep -qi "$downstream=" <<< $ghprbCommentBody + then + downstreamRev[$downstream]=pull/`echo $ghprbCommentBody | sed -r "s/.*${downstream,,}=([0-9]+).*/\1/g"`/merge + fi + done + fi + + # Default to a serial build if no types are given + if test -z "$BTYPES" + then + BTYPES="serial" + fi + + # Convert to arrays for easy looping + BTYPES_ARRAY=($BTYPES) + TOOLCHAINS=($CMAKE_TOOLCHAIN_FILES) +} + + +# Print revisions and configurations +function printHeader { + echo -e "Repository revisions:\n\tIFEM=$IFEM_REVISION" + for upstream in ${upstreams[*]} + do + echo -e "\t$upstream=${upstreamRev[$upstream]}" + done + if [ "$1" != "IFEM" ] + then + echo -e "\t$1=$sha1" + fi + if grep -q "with downstreams" <<< $ghprbCommentBody + then + for downstream in ${downstreams[*]} + do + echo -e "\t$downstream=${downstreamRev[$downstream]}" + done + fi + + echo "Configurations to process: $BTYPES" + echo "Associated toolchain files: $CMAKE_TOOLCHAIN_FILES" +} + + +# $1 = Additional cmake parameters +# $2 = 0 to build and install module, 1 to build and test module +# $3 = Source root of module to build +function build_module { + cmake $3 -DCMAKE_BUILD_TYPE=Release $1 + test $? -eq 0 || exit 1 + # Threaded build + nproc=`nproc` + if test $2 -eq 1 + then + cmake --build . --target testapps -- -j$nproc + test $? -eq 0 || exit 2 + ctest -T Test --test-timeout 180 --no-compress-output + $WORKSPACE/deps/IFEM/jenkins/convert.py -x $WORKSPACE/deps/IFEM/jenkins/conv.xsl -t . > testoutput.xml + else + cmake --build . --target install -- -j$nproc + fi +} + +# $1 = Name of module +# $2 = git-rev to use for module +function clone_module { + # Already cloned by another configuration + if test -d $WORKSPACE/deps/${MODULE_EXTRA_DIR[$1]}$1 + then + return + fi + + pushd . + + mkdir -p $WORKSPACE/deps/${MODULE_EXTRA_DIR[$1]}$1 + cd $WORKSPACE/deps/${MODULE_EXTRA_DIR[$1]}$1 + git init . + + # Hack due to mixed repo locations + declare -A GH_USER + GH_USER[IFEM-AdvectionDiffusion]=SintefMath + GH_USER[IFEM-BeamEx]=SintefMath + GH_USER[IFEM-Elasticity]=OPM + GH_USER[IFEM-FiniteDeformation]=SintefMath + GH_USER[IFEM-NavierStokes]=SintefMath + GH_USER[IFEM-OpenFrac]=OPM + GH_USER[IFEM-PoroElasticity]=OPM + GH_USER[IFEM-Stokes]=SintefMath + GH_USER[IFEM-ThermoElasticity]=SintefMath + + if test -n "$GH_CREDENTIALS" + then + git remote add origin https://$GH_CREDENTIALS@github.com/${GH_USER[$1]}/$1 + else + git remote add origin https://github.com/${GH_USER[$1]}/$1 + fi + git fetch --depth 1 origin $2:branch_to_build + git checkout branch_to_build + test $? -eq 0 || exit 1 + popd +} + + +# $1 = Module to clone +# $2 = Additional cmake parameters +# $3 = git-rev to use for module +# $4 = Build root +function clone_and_build_module { + clone_module $1 $3 + pushd . + mkdir -p $4/build-$1 + cd $4/build-$1 + test_build=0 + if test -n "$5" + then + test_build=$5 + fi + + build_module "$2" $test_build $WORKSPACE/deps/${MODULE_EXTRA_DIR[$1]}$1/${MODULE_APP_DIR[$1]} + test $? -eq 0 || exit 1 + popd +} + + +# $1 - build type +# Uses pre-filled arrays upstreams, and associativ array upstreamRev +# which holds the revisions to use for upstreams. +function build_upstreams { + for upstream in ${upstreams[*]} + do + if grep -qi "$upstream=" <<< $ghprbCommentBody + then + upstreamRev[$upstream]=pull/`echo $ghprbCommentBody | sed -r "s/.*${upstream,,}=([0-9]+).*/\1/g"`/merge + fi + echo "Building upstream $upstream=${upstreamRev[$upstream]} configuration=$1" + + # Build upstream and execute installation + clone_and_build_module $upstream "-DCMAKE_PREFIX_PATH=$WORKSPACE/$1/install -DCMAKE_INSTALL_PREFIX=$WORKSPACE/$1/install -DCMAKE_TOOLCHAIN_FILE=$CMAKE_TOOLCHAIN_FILE" ${upstreamRev[$upstream]} $WORKSPACE/$1 + test $? -eq 0 || exit 1 + done + test $? -eq 0 || exit 1 +} + + +# $1 - name of the module we are called from +# Uses pre-filled arrays downstreams, and associativ array downstreamRev +# which holds the default revisions to use for downstreams +# Uses pre-filled arrays BTYPES_ARRAY and TOOLCHAINS. +function build_downstreams { + for BTYPE in "${!BTYPES_ARRAY[@]}" + do + pushd . + cd $WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/build-$1 + cmake --build . --target install + popd + + egrep_cmd="xml_grep --wrap testsuites --cond testsuite $WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/build-$1/testoutput.xml" + for downstream in ${downstreams[*]} + do + echo "Building downstream $downstream=${downstreamRev[$downstream]} configuration=${BTYPES_ARRAY[$BTYPE]}" + # Build downstream and execute installation + clone_and_build_module $downstream "-DCMAKE_PREFIX_PATH=$WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/install -DCMAKE_INSTALL_PREFIX=$WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/install -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAINS[$BTYPE]}" ${downstreamRev[$downstream]} $WORKSPACE/${BTYPES_ARRAY[$BTYPE]} 1 + test $? -eq 0 || exit 1 + + # Installation for downstream + pushd . + cd $WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/build-$downstream + cmake --build . --target install + popd + egrep_cmd="$egrep_cmd $WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/build-$downstream/testoutput.xml" + done + + $egrep_cmd > $WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/testoutput.xml + test $? -eq 0 || exit 1 + + # Name testsuite according to configuration + sed -i -e "s/classname=\"TestSuite\"/classname=\"${BTYPES_ARRAY[$BTYPE]}\"/g" $WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/testoutput.xml + test $? -eq 0 || exit 1 + done +} + + +# $1 - module name +# Build all upstreams and the context module +# Uses pre-filled arrays BTYPES_ARRAY and TOOLCHAINS. +# Uses variable IFEM_REVISION +function build_module_and_upstreams { + for BTYPE in "${!BTYPES_ARRAY[@]}" + do + if [ "$1" != "IFEM" ] + then + pushd . + mkdir -p ${BTYPES_ARRAY[$BTYPE]}/build-IFEM + cd ${BTYPES_ARRAY[$BTYPE]}/build-IFEM + build_module "-DCMAKE_INSTALL_PREFIX=$WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/install -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAINS[$BTYPE]} -DINSTALL_DOXY=0" 0 $WORKSPACE/deps/IFEM + test $? -eq 0 || exit 1 + popd + + # remove FindIFEM.cmake - causes problems + rm -f ${WORKSPACE}/deps/IFEM/cmake/Modules/FindIFEM.cmake + fi + + CMAKE_TOOLCHAIN_FILE=${TOOLCHAINS[$BTYPE]} + build_upstreams ${BTYPES_ARRAY[$BTYPE]} + + # Build the final module + echo "Building main module $1=$sha1 configuration=${BTYPES_ARRAY[$BTYPE]}" + + # Make a clone + if ! test -d deps/${MODULE_EXTRA_DIR[$1]}$1 + then + mkdir -p deps/${MODULE_EXTRA_DIR[$1]} + git clone $WORKSPACE deps/${MODULE_EXTRA_DIR[$1]}$1 + fi + + # Build + pushd . + mkdir -p ${BTYPES_ARRAY[$BTYPE]}/build-$1 + cd ${BTYPES_ARRAY[$BTYPE]}/build-$1 + build_module "-DCMAKE_INSTALL_PREFIX=$WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/install -DCMAKE_PREFIX_PATH=$WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/install -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" 1 $WORKSPACE/deps/${MODULE_EXTRA_DIR[$1]}$1/${MODULE_APP_DIR[$1]} + test $? -eq 0 || exit 1 + popd + + # Add testsuite names + sed -e "s/classname=\"TestSuite\"/classname=\"${BTYPES_ARRAY[$BTYPE]}\"/g" ${WORKSPACE}/${BTYPES_ARRAY[$BTYPE]}/build-$1/testoutput.xml > ${WORKSPACE}/${BTYPES_ARRAY[$BTYPE]}/testoutput.xml + done +} diff --git a/jenkins/build.sh b/jenkins/build.sh new file mode 100755 index 00000000..5d419e59 --- /dev/null +++ b/jenkins/build.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Downstream revisions +declare -a downstreams +downstreams=(IFEM-Stokes + IFEM-AdvectionDiffusion + IFEM-NavierStokes + IFEM-Elasticity + IFEM-BeamEx + IFEM-FiniteDeformation + IFEM-ThermoElasticity + IFEM-PoroElasticity + IFEM-OpenFrac) + +declare -A downstreamRev +downstreamRev[IFEM-AdvectionDiffusion]=master +downstreamRev[IFEM-BeamEx]=master +downstreamRev[IFEM-Elasticity]=master +downstreamRev[IFEM-FiniteDeformation]=master +downstreamRev[IFEM-NavierStokes]=master +downstreamRev[IFEM-OpenFrac]=master +downstreamRev[IFEM-PoroElasticity]=master +downstreamRev[IFEM-Stokes]=master +downstreamRev[IFEM-ThermoElasticity]=master + +IFEM_REVISION=$sha1 + +source `dirname $0`/build-ifem-module.sh + +parseRevisions +printHeader IFEM + +build_module_and_upstreams IFEM + +# If no downstream builds we are done +if ! grep -q "with downstreams" <<< $ghprbCommentBody +then + exit 0 +fi + +# remove cmake rule so apps do not get confused +mv $WORKSPACE/cmake/Modules/FindIFEM.cmake $WORKSPACE + +build_downstreams IFEM + +# move cmake rule back in place +mv $WORKSPACE/FindIFEM.cmake $WORKSPACE/cmake/Modules + +test $? -eq 0 || exit 1 diff --git a/jenkins/conv.xsl b/jenkins/conv.xsl new file mode 100644 index 00000000..7b946469 --- /dev/null +++ b/jenkins/conv.xsl @@ -0,0 +1,116 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BuildName: + BuildStamp: + Name: + Generator: + CompilerName: + OSName: + Hostname: + OSRelease: + OSVersion: + OSPlatform: + Is64Bits: + VendorString: + VendorID: + FamilyID: + ModelID: + ProcessorCacheSize: + NumberOfLogicalCPU: + NumberOfPhysicalCPU: + TotalVirtualMemory: + TotalPhysicalMemory: + LogicalProcessorsPerPhysical: + ProcessorClockFrequency: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/jenkins/convert.py b/jenkins/convert.py new file mode 100755 index 00000000..c390c33e --- /dev/null +++ b/jenkins/convert.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# coding: utf-8 +# originally from: +# http://www.warp1337.com/content/how-use-ctest-jenkins-xunit-or-junit-plugin +# improved by: +# Jorge Araya Navarro + +# Veni, Sancte Spiritus. + +from lxml import etree +import argparse +from os.path import expanduser +from os.path import join +import logging + +# configure logging +logging.basicConfig(format="%(levelname)s: %(message)s", + level=logging.ERROR) + +desc = ("Converts ctest XML file to xUnit/JUnit XML " + "compatible file to use with Jenkins-CI. " + "Did you found any bug? please report it on: " + "https://bitbucket.org/shackra/ctest-jenkins/issues") + +# configure argument parser. +parser = argparse.ArgumentParser(description=desc) +parser.add_argument("-x", "--xslt", help="the XSLT file to use", required=True) +parser.add_argument("-t", "--tag", help=("the directory where 'Testing/TAG'" + "file is. Remember to call ctest with" + " '-T test' option to generate it"), + required=True) + +parsed = parser.parse_args() +# expanding user symbol "~" +parsed.xsl = expanduser(parsed.xslt) +parsed.tag = expanduser(parsed.tag) + +# opening the TAG file +directory = None +try: + with open(join(parsed.tag, "Testing", "TAG")) as tagfile: + directory = tagfile.readline().strip() + +except NotADirectoryError: + logging.error( + "'Testing/TAG' wasn't found on directory '{}'.".format(parsed.tag)) + exit(1) +except FileNotFoundError: + logging.error( + "File '{}' not found.".format(join(parsed.tag, "Testing", "TAG"))) + exit(1) + +xmldoc = None +transform = None +try: + with open(join(parsed.tag, "Testing", directory, "Test.xml"))\ + as testxmlfile: + xmldoc = etree.parse(testxmlfile) + +except FileNotFoundError: + logging.error("File {} not found. Was it deleted or moved?".format( + join(parsed.tag, "Testing", directory, "Test.xml"))) + exit(1) + +try: + with open(parsed.xslt) as xsltfile: + xslt_root = etree.XML(xsltfile.read()) + transform = etree.XSLT(xslt_root) +except FileNotFoundError: + logging.error("File {} not found.".format(parsed.xslt)) + exit(1) + +result_tree = transform(xmldoc) +print(result_tree)