added: jenkins build scripts with downstream building support

This commit is contained in:
Arne Morten Kvarving 2016-05-26 17:15:09 +02:00
parent 68aed678fb
commit 80f60b9cfa
7 changed files with 523 additions and 2 deletions

View File

@ -17,7 +17,7 @@ mysim=$1
cd `dirname $2`
MAPFILE=`head -n1 $2`
test $? -eq 0 || exit 1
test -n "$3" && mysim="$PETSC_DIR/bin/petscmpiexec -n $3 $mysim"
test -n "$3" && mysim="mpirun -n $3 $mysim"
$mysim $MAPFILE 2>&1 > @CMAKE_BINARY_DIR@/templog
appres=$?
globres=1

View File

@ -41,7 +41,7 @@ macro(IFEM_add_test_app path workdir name)
add_executable(${name}-test EXCLUDE_FROM_ALL ${IFEM_PATH}/src/IFEM-test.C ${TEST_SRCS})
gtest_add_tests($<TARGET_FILE:${name}-test> ${workdir} ${TEST_SRCS})
list(APPEND TEST_APPS ${name}-test)
target_link_libraries(${name}-test ${ARGN} gtest)
target_link_libraries(${name}-test ${ARGN} gtest pthread)
endmacro()
macro(IFEM_add_unittests IFEM_PATH)

30
jenkins/README.md Normal file
View File

@ -0,0 +1,30 @@
# IFEM jenkins build scripts:
**build-ifem-module.sh**:
This is a helper script which contains functions for building,
testing and cloning modules.
**build.sh**:
This expects to run on a jenkins instance with IFEM as the 'origin' remote.
It will build and test IFEM. It can be used both for post-merge builds
of the master branch and for a github pull request builder job.
Optionally you it can build all downstreams and execute their tests.
Important environment variables (jenkins conventions):
WORKSPACE - Root of source tree.
ghprbCommentBody - Trigger as used on github.
GH\_CREDENTIALS - Github credentials to use in the form user:password.
CMAKE\_TOOLCHAIN\_FILES - A space separated list of toolchain files specifying
cmake parameters for a configuration.
BTYPES - A space separated list of build configuration names.
A typically local run building the current content of IFEM and all
downstreams;
WORKSPACE=`pwd` GH\_CREDENTIALS=user:pwd ghprbCommentBody="jenkins build this with downstreams please" jenkins/build.sh

252
jenkins/build-ifem-module.sh Executable file
View File

@ -0,0 +1,252 @@
#!/bin/bash
# Hacks due to not following standards
declare -A MODULE_EXTRA_DIR
MODULE_EXTRA_DIR[IFEM-BeamEx]=IFEM-Elasticity/
MODULE_EXTRA_DIR[IFEM-FiniteDeformation]=IFEM-Elasticity/
declare -A MODULE_APP_DIR
MODULE_APP_DIR[IFEM-BeamEx]=BeamSim
MODULE_APP_DIR[IFEM-Elasticity]=Linear
MODULE_APP_DIR[IFEM-FiniteDeformation]=Nonlinear
# Parse revisions from trigger comment and setup arrays
function parseRevisions {
for upstream in ${upstreams[*]}
do
if grep -qi "$upstream=" <<< $ghprbCommentBody
then
upstreamRev[$upstream]=pull/`echo $ghprbCommentBody | sed -r "s/.*${upstream,,}=([0-9]+).*/\1/g"`/merge
fi
done
if grep -q "with downstreams" <<< $ghprbCommentBody
then
for downstream in ${downstreams[*]}
do
if grep -qi "$downstream=" <<< $ghprbCommentBody
then
downstreamRev[$downstream]=pull/`echo $ghprbCommentBody | sed -r "s/.*${downstream,,}=([0-9]+).*/\1/g"`/merge
fi
done
fi
# Default to a serial build if no types are given
if test -z "$BTYPES"
then
BTYPES="serial"
fi
# Convert to arrays for easy looping
BTYPES_ARRAY=($BTYPES)
TOOLCHAINS=($CMAKE_TOOLCHAIN_FILES)
}
# Print revisions and configurations
function printHeader {
echo -e "Repository revisions:\n\tIFEM=$IFEM_REVISION"
for upstream in ${upstreams[*]}
do
echo -e "\t$upstream=${upstreamRev[$upstream]}"
done
if [ "$1" != "IFEM" ]
then
echo -e "\t$1=$sha1"
fi
if grep -q "with downstreams" <<< $ghprbCommentBody
then
for downstream in ${downstreams[*]}
do
echo -e "\t$downstream=${downstreamRev[$downstream]}"
done
fi
echo "Configurations to process: $BTYPES"
echo "Associated toolchain files: $CMAKE_TOOLCHAIN_FILES"
}
# $1 = Additional cmake parameters
# $2 = 0 to build and install module, 1 to build and test module
# $3 = Source root of module to build
function build_module {
cmake $3 -DCMAKE_BUILD_TYPE=Release $1
test $? -eq 0 || exit 1
# Threaded build
nproc=`nproc`
if test $2 -eq 1
then
cmake --build . --target testapps -- -j$nproc
test $? -eq 0 || exit 2
ctest -T Test --test-timeout 180 --no-compress-output
$WORKSPACE/deps/IFEM/jenkins/convert.py -x $WORKSPACE/deps/IFEM/jenkins/conv.xsl -t . > testoutput.xml
else
cmake --build . --target install -- -j$nproc
fi
}
# $1 = Name of module
# $2 = git-rev to use for module
function clone_module {
# Already cloned by another configuration
if test -d $WORKSPACE/deps/${MODULE_EXTRA_DIR[$1]}$1
then
return
fi
pushd .
mkdir -p $WORKSPACE/deps/${MODULE_EXTRA_DIR[$1]}$1
cd $WORKSPACE/deps/${MODULE_EXTRA_DIR[$1]}$1
git init .
# Hack due to mixed repo locations
declare -A GH_USER
GH_USER[IFEM-AdvectionDiffusion]=SintefMath
GH_USER[IFEM-BeamEx]=SintefMath
GH_USER[IFEM-Elasticity]=OPM
GH_USER[IFEM-FiniteDeformation]=SintefMath
GH_USER[IFEM-NavierStokes]=SintefMath
GH_USER[IFEM-OpenFrac]=OPM
GH_USER[IFEM-PoroElasticity]=OPM
GH_USER[IFEM-Stokes]=SintefMath
GH_USER[IFEM-ThermoElasticity]=SintefMath
if test -n "$GH_CREDENTIALS"
then
git remote add origin https://$GH_CREDENTIALS@github.com/${GH_USER[$1]}/$1
else
git remote add origin https://github.com/${GH_USER[$1]}/$1
fi
git fetch --depth 1 origin $2:branch_to_build
git checkout branch_to_build
test $? -eq 0 || exit 1
popd
}
# $1 = Module to clone
# $2 = Additional cmake parameters
# $3 = git-rev to use for module
# $4 = Build root
function clone_and_build_module {
clone_module $1 $3
pushd .
mkdir -p $4/build-$1
cd $4/build-$1
test_build=0
if test -n "$5"
then
test_build=$5
fi
build_module "$2" $test_build $WORKSPACE/deps/${MODULE_EXTRA_DIR[$1]}$1/${MODULE_APP_DIR[$1]}
test $? -eq 0 || exit 1
popd
}
# $1 - build type
# Uses pre-filled arrays upstreams, and associativ array upstreamRev
# which holds the revisions to use for upstreams.
function build_upstreams {
for upstream in ${upstreams[*]}
do
if grep -qi "$upstream=" <<< $ghprbCommentBody
then
upstreamRev[$upstream]=pull/`echo $ghprbCommentBody | sed -r "s/.*${upstream,,}=([0-9]+).*/\1/g"`/merge
fi
echo "Building upstream $upstream=${upstreamRev[$upstream]} configuration=$1"
# Build upstream and execute installation
clone_and_build_module $upstream "-DCMAKE_PREFIX_PATH=$WORKSPACE/$1/install -DCMAKE_INSTALL_PREFIX=$WORKSPACE/$1/install -DCMAKE_TOOLCHAIN_FILE=$CMAKE_TOOLCHAIN_FILE" ${upstreamRev[$upstream]} $WORKSPACE/$1
test $? -eq 0 || exit 1
done
test $? -eq 0 || exit 1
}
# $1 - name of the module we are called from
# Uses pre-filled arrays downstreams, and associativ array downstreamRev
# which holds the default revisions to use for downstreams
# Uses pre-filled arrays BTYPES_ARRAY and TOOLCHAINS.
function build_downstreams {
for BTYPE in "${!BTYPES_ARRAY[@]}"
do
pushd .
cd $WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/build-$1
cmake --build . --target install
popd
egrep_cmd="xml_grep --wrap testsuites --cond testsuite $WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/build-$1/testoutput.xml"
for downstream in ${downstreams[*]}
do
echo "Building downstream $downstream=${downstreamRev[$downstream]} configuration=${BTYPES_ARRAY[$BTYPE]}"
# Build downstream and execute installation
clone_and_build_module $downstream "-DCMAKE_PREFIX_PATH=$WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/install -DCMAKE_INSTALL_PREFIX=$WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/install -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAINS[$BTYPE]}" ${downstreamRev[$downstream]} $WORKSPACE/${BTYPES_ARRAY[$BTYPE]} 1
test $? -eq 0 || exit 1
# Installation for downstream
pushd .
cd $WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/build-$downstream
cmake --build . --target install
popd
egrep_cmd="$egrep_cmd $WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/build-$downstream/testoutput.xml"
done
$egrep_cmd > $WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/testoutput.xml
test $? -eq 0 || exit 1
# Name testsuite according to configuration
sed -i -e "s/classname=\"TestSuite\"/classname=\"${BTYPES_ARRAY[$BTYPE]}\"/g" $WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/testoutput.xml
test $? -eq 0 || exit 1
done
}
# $1 - module name
# Build all upstreams and the context module
# Uses pre-filled arrays BTYPES_ARRAY and TOOLCHAINS.
# Uses variable IFEM_REVISION
function build_module_and_upstreams {
for BTYPE in "${!BTYPES_ARRAY[@]}"
do
if [ "$1" != "IFEM" ]
then
pushd .
mkdir -p ${BTYPES_ARRAY[$BTYPE]}/build-IFEM
cd ${BTYPES_ARRAY[$BTYPE]}/build-IFEM
build_module "-DCMAKE_INSTALL_PREFIX=$WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/install -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAINS[$BTYPE]} -DINSTALL_DOXY=0" 0 $WORKSPACE/deps/IFEM
test $? -eq 0 || exit 1
popd
# remove FindIFEM.cmake - causes problems
rm -f ${WORKSPACE}/deps/IFEM/cmake/Modules/FindIFEM.cmake
fi
CMAKE_TOOLCHAIN_FILE=${TOOLCHAINS[$BTYPE]}
build_upstreams ${BTYPES_ARRAY[$BTYPE]}
# Build the final module
echo "Building main module $1=$sha1 configuration=${BTYPES_ARRAY[$BTYPE]}"
# Make a clone
if ! test -d deps/${MODULE_EXTRA_DIR[$1]}$1
then
mkdir -p deps/${MODULE_EXTRA_DIR[$1]}
git clone $WORKSPACE deps/${MODULE_EXTRA_DIR[$1]}$1
fi
# Build
pushd .
mkdir -p ${BTYPES_ARRAY[$BTYPE]}/build-$1
cd ${BTYPES_ARRAY[$BTYPE]}/build-$1
build_module "-DCMAKE_INSTALL_PREFIX=$WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/install -DCMAKE_PREFIX_PATH=$WORKSPACE/${BTYPES_ARRAY[$BTYPE]}/install -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" 1 $WORKSPACE/deps/${MODULE_EXTRA_DIR[$1]}$1/${MODULE_APP_DIR[$1]}
test $? -eq 0 || exit 1
popd
# Add testsuite names
sed -e "s/classname=\"TestSuite\"/classname=\"${BTYPES_ARRAY[$BTYPE]}\"/g" ${WORKSPACE}/${BTYPES_ARRAY[$BTYPE]}/build-$1/testoutput.xml > ${WORKSPACE}/${BTYPES_ARRAY[$BTYPE]}/testoutput.xml
done
}

49
jenkins/build.sh Executable file
View File

@ -0,0 +1,49 @@
#!/bin/bash
# Downstream revisions
declare -a downstreams
downstreams=(IFEM-Stokes
IFEM-AdvectionDiffusion
IFEM-NavierStokes
IFEM-Elasticity
IFEM-BeamEx
IFEM-FiniteDeformation
IFEM-ThermoElasticity
IFEM-PoroElasticity
IFEM-OpenFrac)
declare -A downstreamRev
downstreamRev[IFEM-AdvectionDiffusion]=master
downstreamRev[IFEM-BeamEx]=master
downstreamRev[IFEM-Elasticity]=master
downstreamRev[IFEM-FiniteDeformation]=master
downstreamRev[IFEM-NavierStokes]=master
downstreamRev[IFEM-OpenFrac]=master
downstreamRev[IFEM-PoroElasticity]=master
downstreamRev[IFEM-Stokes]=master
downstreamRev[IFEM-ThermoElasticity]=master
IFEM_REVISION=$sha1
source `dirname $0`/build-ifem-module.sh
parseRevisions
printHeader IFEM
build_module_and_upstreams IFEM
# If no downstream builds we are done
if ! grep -q "with downstreams" <<< $ghprbCommentBody
then
exit 0
fi
# remove cmake rule so apps do not get confused
mv $WORKSPACE/cmake/Modules/FindIFEM.cmake $WORKSPACE
build_downstreams IFEM
# move cmake rule back in place
mv $WORKSPACE/FindIFEM.cmake $WORKSPACE/cmake/Modules
test $? -eq 0 || exit 1

116
jenkins/conv.xsl Normal file
View File

@ -0,0 +1,116 @@
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
<xsl:output method="xml" indent="yes"/>
<xsl:template match="/Site">
<testsuite>
<xsl:variable name="BuildName"><xsl:value-of select="@BuildName"/></xsl:variable>
<xsl:variable name="BuildStamp"><xsl:value-of select="@BuildStamp"/></xsl:variable>
<xsl:variable name="Name"><xsl:value-of select="@Name"/></xsl:variable>
<xsl:variable name="Generator"><xsl:value-of select="@Generator"/></xsl:variable>
<xsl:variable name="CompilerName"><xsl:value-of select="@CompilerName"/></xsl:variable>
<xsl:variable name="OSName"><xsl:value-of select="@OSName"/></xsl:variable>
<xsl:variable name="Hostname"><xsl:value-of select="@Hostname"/></xsl:variable>
<xsl:variable name="OSRelease"><xsl:value-of select="@OSRelease"/></xsl:variable>
<xsl:variable name="OSVersion"><xsl:value-of select="@OSVersion"/></xsl:variable>
<xsl:variable name="OSPlatform"><xsl:value-of select="@OSPlatform"/></xsl:variable>
<xsl:variable name="Is64Bits"><xsl:value-of select="@Is64Bits"/></xsl:variable>
<xsl:variable name="VendorString"><xsl:value-of select="@VendorString"/></xsl:variable>
<xsl:variable name="VendorID"><xsl:value-of select="@VendorID"/></xsl:variable>
<xsl:variable name="FamilyID"><xsl:value-of select="@FamilyID"/></xsl:variable>
<xsl:variable name="ModelID"><xsl:value-of select="@ModelID"/></xsl:variable>
<xsl:variable name="ProcessorCacheSize"><xsl:value-of select="@ProcessorCacheSize"/></xsl:variable>
<xsl:variable name="NumberOfLogicalCPU"><xsl:value-of select="@NumberOfLogicalCPU"/></xsl:variable>
<xsl:variable name="NumberOfPhysicalCPU"><xsl:value-of select="@NumberOfPhysicalCPU"/></xsl:variable>
<xsl:variable name="TotalVirtualMemory"><xsl:value-of select="@TotalVirtualMemory"/></xsl:variable>
<xsl:variable name="TotalPhysicalMemory"><xsl:value-of select="@TotalPhysicalMemory"/></xsl:variable>
<xsl:variable name="LogicalProcessorsPerPhysical"><xsl:value-of select="@LogicalProcessorsPerPhysical"/></xsl:variable>
<xsl:variable name="ProcessorClockFrequency"><xsl:value-of select="@ProcessorClockFrequency"/></xsl:variable>
<properties>
<property name="BuildName" value="{$BuildName}" />
<property name="BuildStamp" value="{$BuildStamp}" />
<property name="Name" value="{$Name}" />
<property name="Generator" value="{$Generator}" />
<property name="CompilerName" value="{$CompilerName}" />
<property name="OSName" value="{$OSName}" />
<property name="Hostname" value="{$Hostname}" />
<property name="OSRelease" value="{$OSRelease}" />
<property name="OSVersion" value="{$OSVersion}" />
<property name="OSPlatform" value="{$OSPlatform}" />
<property name="Is64Bits" value="{$Is64Bits}" />
<property name="VendorString" value="{$VendorString}" />
<property name="VendorID" value="{$VendorID}" />
<property name="FamilyID" value="{$FamilyID}" />
<property name="ModelID" value="{$ModelID}" />
<property name="ProcessorCacheSize" value="{$ProcessorCacheSize}" />
<property name="NumberOfLogicalCPU" value="{$NumberOfLogicalCPU}" />
<property name="NumberOfPhysicalCPU" value="{$NumberOfPhysicalCPU}" />
<property name="TotalVirtualMemory" value="{$TotalVirtualMemory}" />
<property name="TotalPhysicalMemory" value="{$TotalPhysicalMemory}" />
<property name="LogicalProcessorsPerPhysical" value="{$LogicalProcessorsPerPhysical}" />
<property name="ProcessorClockFrequency" value="{$ProcessorClockFrequency}" />
</properties>
<xsl:apply-templates select="Testing/Test"/>
<system-out>
BuildName: <xsl:value-of select="$BuildName" />
BuildStamp: <xsl:value-of select="$BuildStamp" />
Name: <xsl:value-of select="$Name" />
Generator: <xsl:value-of select="$Generator" />
CompilerName: <xsl:value-of select="$CompilerName" />
OSName: <xsl:value-of select="$OSName" />
Hostname: <xsl:value-of select="$Hostname" />
OSRelease: <xsl:value-of select="$OSRelease" />
OSVersion: <xsl:value-of select="$OSVersion" />
OSPlatform: <xsl:value-of select="$OSPlatform" />
Is64Bits: <xsl:value-of select="$Is64Bits" />
VendorString: <xsl:value-of select="$VendorString" />
VendorID: <xsl:value-of select="$VendorID" />
FamilyID: <xsl:value-of select="$FamilyID" />
ModelID: <xsl:value-of select="$ModelID" />
ProcessorCacheSize: <xsl:value-of select="$ProcessorCacheSize" />
NumberOfLogicalCPU: <xsl:value-of select="$NumberOfLogicalCPU" />
NumberOfPhysicalCPU: <xsl:value-of select="$NumberOfPhysicalCPU" />
TotalVirtualMemory: <xsl:value-of select="$TotalVirtualMemory" />
TotalPhysicalMemory: <xsl:value-of select="$TotalPhysicalMemory" />
LogicalProcessorsPerPhysical: <xsl:value-of select="$LogicalProcessorsPerPhysical" />
ProcessorClockFrequency: <xsl:value-of select="$ProcessorClockFrequency" />
</system-out>
</testsuite>
</xsl:template>
<xsl:template match="Testing/Test">
<xsl:variable name="testcasename"><xsl:value-of select= "Name"/></xsl:variable>
<xsl:variable name="exectime">
<xsl:for-each select="Results/NamedMeasurement">
<xsl:if test="@name = 'Execution Time'">
<xsl:value-of select="."/>
</xsl:if>
</xsl:for-each>
</xsl:variable>
<testcase name="{$testcasename}" classname="TestSuite" time="{$exectime}">
<xsl:if test="@Status = 'passed'">
</xsl:if>
<xsl:if test="@Status = 'failed'">
<xsl:variable name="failtype">
<xsl:for-each select="Results/NamedMeasurement">
<xsl:if test="@name = 'Exit Code'">
<xsl:value-of select="."/>
</xsl:if>
</xsl:for-each>
</xsl:variable>
<xsl:variable name="failcode">
<xsl:for-each select="Results/NamedMeasurement">
<xsl:if test="@name = 'Exit Value'">
<xsl:value-of select="."/>
</xsl:if>
</xsl:for-each>
</xsl:variable>
<error message="{$failtype} ({$failcode})"><xsl:value-of select="Results/Measurement/Value/text()" /></error>
</xsl:if>
<xsl:if test="@Status = 'notrun'">
<skipped><xsl:value-of select="Results/Measurement/Value/text()" /></skipped>
</xsl:if>
</testcase>
</xsl:template>
</xsl:stylesheet>

74
jenkins/convert.py Executable file
View File

@ -0,0 +1,74 @@
#!/usr/bin/env python
# coding: utf-8
# originally from:
# http://www.warp1337.com/content/how-use-ctest-jenkins-xunit-or-junit-plugin
# improved by:
# Jorge Araya Navarro <elcorreo@deshackra.com>
# Veni, Sancte Spiritus.
from lxml import etree
import argparse
from os.path import expanduser
from os.path import join
import logging
# configure logging
logging.basicConfig(format="%(levelname)s: %(message)s",
level=logging.ERROR)
desc = ("Converts ctest XML file to xUnit/JUnit XML "
"compatible file to use with Jenkins-CI. "
"Did you found any bug? please report it on: "
"https://bitbucket.org/shackra/ctest-jenkins/issues")
# configure argument parser.
parser = argparse.ArgumentParser(description=desc)
parser.add_argument("-x", "--xslt", help="the XSLT file to use", required=True)
parser.add_argument("-t", "--tag", help=("the directory where 'Testing/TAG'"
"file is. Remember to call ctest with"
" '-T test' option to generate it"),
required=True)
parsed = parser.parse_args()
# expanding user symbol "~"
parsed.xsl = expanduser(parsed.xslt)
parsed.tag = expanduser(parsed.tag)
# opening the TAG file
directory = None
try:
with open(join(parsed.tag, "Testing", "TAG")) as tagfile:
directory = tagfile.readline().strip()
except NotADirectoryError:
logging.error(
"'Testing/TAG' wasn't found on directory '{}'.".format(parsed.tag))
exit(1)
except FileNotFoundError:
logging.error(
"File '{}' not found.".format(join(parsed.tag, "Testing", "TAG")))
exit(1)
xmldoc = None
transform = None
try:
with open(join(parsed.tag, "Testing", directory, "Test.xml"))\
as testxmlfile:
xmldoc = etree.parse(testxmlfile)
except FileNotFoundError:
logging.error("File {} not found. Was it deleted or moved?".format(
join(parsed.tag, "Testing", directory, "Test.xml")))
exit(1)
try:
with open(parsed.xslt) as xsltfile:
xslt_root = etree.XML(xsltfile.read())
transform = etree.XSLT(xslt_root)
except FileNotFoundError:
logging.error("File {} not found.".format(parsed.xslt))
exit(1)
result_tree = transform(xmldoc)
print(result_tree)