mirror of
https://github.com/OPM/opm-simulators.git
synced 2025-02-25 18:55:30 -06:00
Merge pull request #1045 from akva2/more_summary_analysis
[WIP] Perform more analysis of summary regression test failures
This commit is contained in:
commit
cac6a33a51
69
tests/analyze_ecl_failure.sh
Executable file
69
tests/analyze_ecl_failure.sh
Executable file
@ -0,0 +1,69 @@
|
||||
#!/bin/bash
|
||||
# This will perform some analysis on a failed restart/init
|
||||
# comparisons in a regression tests.
|
||||
|
||||
# Analyze restart/init test failure.
|
||||
# - Print failed keywords
|
||||
# - Print maximum deviations for each keyword
|
||||
analyzeEclFailure() {
|
||||
kwds=`cat $KWTMPFILE | tr '\n' ' ' | sed -e 's/.*Common keywords for the two cases:\(.*\)Uncommon.*/\1/'`
|
||||
TMPFILE=$(mktemp)
|
||||
allok=1
|
||||
for kwd in $kwds
|
||||
do
|
||||
${COMPARE_ECL_COMMAND} -n -l -k ${kwd} ${FILE1} ${FILE2} ${ABS_TOL} ${REL_TOL} &> ${TMPFILE}
|
||||
nfailure=`cat ${TMPFILE} | grep "Deviations exceed tolerances" | wc -l`
|
||||
if [ ${nfailure} -ne 0 ]
|
||||
then
|
||||
allok=0
|
||||
echo "Failure for keyword ${kwd}"
|
||||
echo -e "\t Fails for ${nfailure} entries"
|
||||
alines=`cat $TMPFILE | grep "The absolute deviation is"`
|
||||
IFS=$'\n'
|
||||
abs=0
|
||||
for line in $alines
|
||||
do
|
||||
abs_new=`echo $line| awk -F ' ' '{print $5}'`
|
||||
abs_new=`echo ${abs_new: : -1} | awk '{printf sprintf("%.16f", $1); }'`
|
||||
if [ `bc <<< "$abs_new>$abs"` -eq 1 ]
|
||||
then
|
||||
abs=$abs_new
|
||||
fi
|
||||
done
|
||||
rlines=`cat $TMPFILE | grep "The relative deviation is"`
|
||||
rel=0
|
||||
for line in $rlines
|
||||
do
|
||||
rel_new=`echo $line| awk -F ' ' '{print $5}'`
|
||||
rel_new=`echo ${rel_new: : -1} | awk '{printf sprintf("%.16f", $1); }'`
|
||||
if [ `bc <<< "$rel_new>$rel"` -eq 1 ]
|
||||
then
|
||||
rel=$rel_new
|
||||
fi
|
||||
done
|
||||
echo -e "\t Largest absolute deviation: `echo $abs | awk '{printf sprintf("%e", $1); }'`"
|
||||
echo -e "\t Largest relative deviation: `echo $rel | awk '{printf sprintf("%e", $1); }'`"
|
||||
fi
|
||||
done
|
||||
if [ $allok -eq 1 ]
|
||||
then
|
||||
echo "Comparisons pass for all common keywords."
|
||||
fi
|
||||
}
|
||||
|
||||
COMPARE_ECL_COMMAND=$1
|
||||
TYPE=$2
|
||||
FILE1=$3
|
||||
FILE2=$4
|
||||
ABS_TOL=$5
|
||||
REL_TOL=$6
|
||||
|
||||
KWTMPFILE=$(mktemp)
|
||||
${COMPARE_ECL_COMMAND} -t ${TYPE} -l -P ${FILE1} ${FILE2} ${ABS_TOL} ${REL_TOL} &> $KWTMPFILE
|
||||
|
||||
cat $KWTMPFILE
|
||||
echo ""
|
||||
|
||||
analyzeEclFailure
|
||||
|
||||
rm -f $KWTMPFILE
|
63
tests/analyze_summary_failure.sh
Executable file
63
tests/analyze_summary_failure.sh
Executable file
@ -0,0 +1,63 @@
|
||||
#!/bin/bash
|
||||
# This will perform some analysis on a failed summary
|
||||
# comparisons in a regression tests.
|
||||
|
||||
# Analyze summary test failure.
|
||||
# - Print number of failed keywords
|
||||
# - Print number of levels each keyword failed at
|
||||
# - Print maximum deviations for each keyword
|
||||
analyzeSummaryFailure() {
|
||||
lines=`cat $TMPFILE | grep "For keyword"`
|
||||
IFS=$'\n'
|
||||
kwds=""
|
||||
for line in $lines
|
||||
do
|
||||
kwds+="`echo $line | awk -F ' ' '{print $3}'`\n"
|
||||
done
|
||||
unique_kwds=`echo -e $kwds | uniq`
|
||||
kws_failed=`echo -e "$unique_kwds" | wc -l`
|
||||
echo "$kws_failed summary keyword(s) exhibit failures"
|
||||
numsteps=`cat $TMPFILE | grep "Comparing " | grep steps | awk -F ' ' '{print $2}'`
|
||||
|
||||
for kwd in $unique_kwds
|
||||
do
|
||||
lines=`cat $TMPFILE | grep -n "For keyword $kwd$"`
|
||||
echo -e "\t $kwd"
|
||||
echo -e "\t \t Fails for: `cat $TMPFILE | grep -n "$kwd$" | wc -l` / $numsteps steps."
|
||||
abs=0
|
||||
rel=0
|
||||
for line in $lines
|
||||
do
|
||||
ln=`echo $line | awk -F ':' '{print $1'}`
|
||||
abs_line=$(($ln+2))
|
||||
rel_line=$(($ln+3))
|
||||
abs_new=`sed "${abs_line}q;d" $TMPFILE | awk -F ' ' '{print $5}'`
|
||||
rel_new=`sed "${rel_line}q;d" $TMPFILE | awk -F ' ' '{print $5}'`
|
||||
abs_new=`echo ${abs_new: : -1} | awk '{printf sprintf("%.16f", $1); }'`
|
||||
rel_new=`echo ${rel_new: : -1} | awk '{printf sprintf("%.16f", $1); }'`
|
||||
if [ `bc <<< "$abs_new>$abs"` -eq 1 ]
|
||||
then
|
||||
abs=$abs_new
|
||||
fi
|
||||
if [ `bc <<< "$rel_new>$rel"` -eq 1 ]
|
||||
then
|
||||
rel=$rel_new
|
||||
fi
|
||||
done
|
||||
echo -e "\t\t Largest absolute error: `echo $abs | awk '{printf sprintf("%e", $1); }'`"
|
||||
echo -e "\t\t Largest relative error: `echo $rel | awk '{printf sprintf("%e", $1); }'`"
|
||||
done
|
||||
}
|
||||
|
||||
COMPARE_SUMMARY_COMMAND=$1
|
||||
PARAM=$2
|
||||
FILE1=$3
|
||||
FILE2=$4
|
||||
ABS_TOL=$5
|
||||
REL_TOL=$6
|
||||
|
||||
TMPFILE=$(mktemp)
|
||||
${COMPARE_SUMMARY_COMMAND} -p -n ${PARAM} ${FILE1} ${FILE2} ${ABS_TOL} ${REL_TOL} &> $TMPFILE
|
||||
|
||||
analyzeSummaryFailure
|
||||
rm -f $TMPFILE
|
@ -1,5 +1,10 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# This runs the initialization step of a simulator,
|
||||
# then compares the resulting INIT file against a reference.
|
||||
# This is meant to track regressions in INIT file writing.
|
||||
# Useful for models that are too large to do simulation on
|
||||
# as a regression test.
|
||||
|
||||
INPUT_DATA_PATH="$1"
|
||||
RESULT_PATH="$2"
|
||||
@ -19,4 +24,12 @@ cd ${RESULT_PATH}
|
||||
${BINPATH}/${EXE_NAME} ${TEST_ARGS} nosim=true
|
||||
cd ..
|
||||
|
||||
ecode=0
|
||||
${COMPARE_ECL_COMMAND} -t INIT ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
ecode=1
|
||||
`dirname $0`/analyze_ecl_failure.sh ${COMPARE_ECL_COMMAND} INIT ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
|
||||
fi
|
||||
|
||||
exit $ecode
|
||||
|
@ -1,5 +1,8 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# This performs a serial and a parallel for a simulator,
|
||||
# then compares the summary and restart files from the two runs.
|
||||
# Meant to track regression in parallel simulators.
|
||||
|
||||
INPUT_DATA_PATH="$1"
|
||||
RESULT_PATH="$2"
|
||||
@ -17,10 +20,25 @@ rm -Rf ${RESULT_PATH}
|
||||
mkdir -p ${RESULT_PATH}
|
||||
cd ${RESULT_PATH}
|
||||
${BINPATH}/${EXE_NAME} ${TEST_ARGS}.DATA linear_solver_reduction=1e-7 tolerance_cnv=5e-6 tolerance_mb=1e-8
|
||||
test $? -eq 0 || exit 1
|
||||
mkdir mpi
|
||||
cd mpi
|
||||
mpirun -np 4 ${BINPATH}/${EXE_NAME} ${TEST_ARGS}.DATA linear_solver_reduction=1e-7 tolerance_cnv=5e-6 tolerance_mb=1e-8
|
||||
test $? -eq 0 || exit 1
|
||||
cd ..
|
||||
|
||||
ecode=0
|
||||
${COMPARE_SUMMARY_COMMAND} -R ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL}
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
ecode=1
|
||||
`dirname $0`/analyze_summary_failure.sh ${COMPARE_SUMMARY_COMMAND} -R ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL}
|
||||
fi
|
||||
${COMPARE_ECL_COMMAND} -l ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL}
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
ecode=1
|
||||
`dirname $0`/analyze_ecl_failure.sh ${COMPARE_ECL_COMMAND} UNRST ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL}
|
||||
fi
|
||||
|
||||
exit $ecode
|
||||
|
@ -1,5 +1,7 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# This runs a simulator, then compares the summary, restart and init
|
||||
# files against a reference.
|
||||
|
||||
INPUT_DATA_PATH="$1"
|
||||
RESULT_PATH="$2"
|
||||
@ -19,8 +21,26 @@ cd ${RESULT_PATH}
|
||||
${BINPATH}/${EXE_NAME} ${TEST_ARGS}
|
||||
cd ..
|
||||
|
||||
ecode=0
|
||||
${COMPARE_SUMMARY_COMMAND} -r ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
ecode=1
|
||||
`dirname $0`/analyze_summary_failure.sh ${COMPARE_SUMMARY_COMMAND} -r ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
|
||||
fi
|
||||
|
||||
${COMPARE_ECL_COMMAND} ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
ecode=1
|
||||
`dirname $0`/analyze_ecl_failure.sh ${COMPARE_ECL_COMMAND} UNRST ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
|
||||
fi
|
||||
|
||||
${COMPARE_ECL_COMMAND} -t INIT ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
ecode=1
|
||||
`dirname $0`/analyze_ecl_failure.sh ${COMPARE_ECL_COMMAND} INIT ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
|
||||
fi
|
||||
|
||||
exit $ecode
|
||||
|
@ -1,5 +1,8 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# This runs a simulator from start to end, then a restarted
|
||||
# run of the simulator, before comparing the output from the two runs.
|
||||
# This is meant to track regressions in the restart support.
|
||||
|
||||
INPUT_DATA_PATH="$1"
|
||||
RESULT_PATH="$2"
|
||||
@ -17,8 +20,23 @@ rm -Rf ${RESULT_PATH}
|
||||
mkdir -p ${RESULT_PATH}
|
||||
cd ${RESULT_PATH}
|
||||
${BINPATH}/${EXE_NAME} ${TEST_ARGS}.DATA timestep.adaptive=false
|
||||
test $? -eq 0 || exit 1
|
||||
${BINPATH}/${EXE_NAME} ${TEST_ARGS}_RESTART.DATA timestep.adaptive=false
|
||||
test $? -eq 0 || exit 1
|
||||
|
||||
ecode=0
|
||||
${COMPARE_SUMMARY_COMMAND} -R ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/${FILENAME}_RESTART ${ABS_TOL} ${REL_TOL}
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
ecode=1
|
||||
`dirname $0`/analyze_summary_failure.sh ${COMPARE_SUMMARY_COMMAND} -R ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/${FILENAME}_RESTART ${ABS_TOL} ${REL_TOL}
|
||||
fi
|
||||
|
||||
${COMPARE_ECL_COMMAND} -l ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/${FILENAME}_RESTART ${ABS_TOL} ${REL_TOL}
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
ecode=1
|
||||
`dirname $0`/analyze_ecl_failure.sh ${COMPARE_ECL_COMMAND} UNRST ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/${FILENAME}_RESTART ${ABS_TOL} ${REL_TOL}
|
||||
fi
|
||||
|
||||
exit $ecode
|
||||
|
Loading…
Reference in New Issue
Block a user