From d2730175a0ebbb8df85615192f47e944652d95a9 Mon Sep 17 00:00:00 2001
From: Arne Morten Kvarving <arne.morten.kvarving@sintef.no>
Date: Thu, 2 Feb 2017 14:58:23 +0100
Subject: [PATCH 1/2] changed: perform all comparisons in regression tests in
 all cases

we want to process all comparisons, even if a previous comparison
failed. will aid further analysis
---
 tests/run-parallel-regressionTest.sh | 11 ++++++++++-
 tests/run-regressionTest.sh          | 10 +++++++++-
 tests/run-restart-regressionTest.sh  | 11 ++++++++++-
 3 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/tests/run-parallel-regressionTest.sh b/tests/run-parallel-regressionTest.sh
index 445f1258f..7d7a7209e 100755
--- a/tests/run-parallel-regressionTest.sh
+++ b/tests/run-parallel-regressionTest.sh
@@ -1,5 +1,4 @@
 #!/bin/bash
-set -e
 
 INPUT_DATA_PATH="$1"
 RESULT_PATH="$2"
@@ -17,10 +16,20 @@ rm -Rf ${RESULT_PATH}
 mkdir -p ${RESULT_PATH}
 cd ${RESULT_PATH}
 ${BINPATH}/${EXE_NAME} ${TEST_ARGS}.DATA linear_solver_reduction=1e-7 tolerance_cnv=5e-6 tolerance_mb=1e-8
+test $? -eq 0 || exit 1
 mkdir mpi
 cd mpi
 mpirun -np 4 ${BINPATH}/${EXE_NAME} ${TEST_ARGS}.DATA linear_solver_reduction=1e-7 tolerance_cnv=5e-6 tolerance_mb=1e-8
+test $? -eq 0 || exit 1
 cd ..
 
+ecode=0
 ${COMPARE_SUMMARY_COMMAND} -R ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL}
+if [ $? -ne 0 ]
+then
+  ecode=1
+fi
 ${COMPARE_ECL_COMMAND} -l ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL}
+test $? -eq 0 || ecode=1
+
+exit $ecode
diff --git a/tests/run-regressionTest.sh b/tests/run-regressionTest.sh
index d2705eca6..ce00f6e75 100755
--- a/tests/run-regressionTest.sh
+++ b/tests/run-regressionTest.sh
@@ -1,5 +1,4 @@
 #!/bin/bash
-set -e
 
 INPUT_DATA_PATH="$1"
 RESULT_PATH="$2"
@@ -19,8 +18,17 @@ cd ${RESULT_PATH}
 ${BINPATH}/${EXE_NAME} ${TEST_ARGS}
 cd ..
 
+ecode=0
 ${COMPARE_SUMMARY_COMMAND} -r ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
+if [ $? -ne 0 ]
+then
+  ecode=1
+fi
 
 ${COMPARE_ECL_COMMAND}  ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
+test $? -eq 0 || ecode=1
 
 ${COMPARE_ECL_COMMAND} -t INIT ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
+test $? -eq 0 || ecode=1
+
+exit $ecode
diff --git a/tests/run-restart-regressionTest.sh b/tests/run-restart-regressionTest.sh
index 67f7b09ef..3fcd5cf38 100755
--- a/tests/run-restart-regressionTest.sh
+++ b/tests/run-restart-regressionTest.sh
@@ -1,5 +1,4 @@
 #!/bin/bash
-set -e
 
 INPUT_DATA_PATH="$1"
 RESULT_PATH="$2"
@@ -17,8 +16,18 @@ rm -Rf ${RESULT_PATH}
 mkdir -p ${RESULT_PATH}
 cd ${RESULT_PATH}
 ${BINPATH}/${EXE_NAME} ${TEST_ARGS}.DATA timestep.adaptive=false
+test $? -eq 0 || exit 1
 ${BINPATH}/${EXE_NAME} ${TEST_ARGS}_RESTART.DATA timestep.adaptive=false
+test $? -eq 0 || exit 1
 
+ecode=0
 ${COMPARE_SUMMARY_COMMAND} -R ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/${FILENAME}_RESTART ${ABS_TOL} ${REL_TOL}
+if [ $? -ne 0 ]
+then
+  ecode=1
+fi
 
 ${COMPARE_ECL_COMMAND} -l ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/${FILENAME}_RESTART ${ABS_TOL} ${REL_TOL}
+test $? -eq 0 || ecode=1
+
+exit $ecode

From 6183e78402167b82d17bac836b825065431adf28 Mon Sep 17 00:00:00 2001
From: Arne Morten Kvarving <arne.morten.kvarving@sintef.no>
Date: Thu, 16 Feb 2017 16:00:42 +0100
Subject: [PATCH 2/2] perform some more analysis on regression test failures

---
 tests/analyze_ecl_failure.sh         | 69 ++++++++++++++++++++++++++++
 tests/analyze_summary_failure.sh     | 63 +++++++++++++++++++++++++
 tests/run-init-regressionTest.sh     | 15 +++++-
 tests/run-parallel-regressionTest.sh | 11 ++++-
 tests/run-regressionTest.sh          | 16 ++++++-
 tests/run-restart-regressionTest.sh  | 11 ++++-
 6 files changed, 180 insertions(+), 5 deletions(-)
 create mode 100755 tests/analyze_ecl_failure.sh
 create mode 100755 tests/analyze_summary_failure.sh

diff --git a/tests/analyze_ecl_failure.sh b/tests/analyze_ecl_failure.sh
new file mode 100755
index 000000000..39a3d92e9
--- /dev/null
+++ b/tests/analyze_ecl_failure.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+# This will perform some analysis on a failed restart/init
+# comparisons in a regression tests.
+
+# Analyze restart/init test failure.
+# - Print failed keywords
+# - Print maximum deviations for each keyword
+analyzeEclFailure() {
+kwds=`cat $KWTMPFILE | tr '\n' ' ' | sed -e 's/.*Common keywords for the two cases:\(.*\)Uncommon.*/\1/'`
+TMPFILE=$(mktemp)
+allok=1
+for kwd in $kwds
+do
+  ${COMPARE_ECL_COMMAND} -n -l -k ${kwd} ${FILE1} ${FILE2} ${ABS_TOL} ${REL_TOL} &> ${TMPFILE}
+  nfailure=`cat ${TMPFILE} | grep "Deviations exceed tolerances" | wc -l`
+  if [ ${nfailure} -ne 0 ]
+  then
+    allok=0
+    echo "Failure for keyword ${kwd}"
+    echo -e "\t Fails for ${nfailure} entries"
+    alines=`cat $TMPFILE | grep "The absolute deviation is"`
+    IFS=$'\n'
+    abs=0
+    for line in $alines
+    do
+      abs_new=`echo $line| awk -F ' ' '{print $5}'`
+      abs_new=`echo ${abs_new: : -1} | awk '{printf sprintf("%.16f", $1); }'`
+      if [ `bc <<< "$abs_new>$abs"` -eq 1 ]
+      then
+        abs=$abs_new
+      fi
+    done
+    rlines=`cat $TMPFILE | grep "The relative deviation is"`
+    rel=0
+    for line in $rlines
+    do
+      rel_new=`echo $line| awk -F ' ' '{print $5}'`
+      rel_new=`echo ${rel_new: : -1} | awk '{printf sprintf("%.16f", $1); }'`
+      if [ `bc <<< "$rel_new>$rel"` -eq 1 ]
+      then
+        rel=$rel_new
+      fi
+    done
+    echo -e "\t Largest absolute deviation: `echo $abs | awk '{printf sprintf("%e", $1); }'`"
+    echo -e "\t Largest relative deviation: `echo $rel | awk '{printf sprintf("%e", $1); }'`"
+  fi
+done
+if [ $allok -eq 1 ]
+then
+  echo "Comparisons pass for all common keywords."
+fi
+}
+
+COMPARE_ECL_COMMAND=$1
+TYPE=$2
+FILE1=$3
+FILE2=$4
+ABS_TOL=$5
+REL_TOL=$6
+
+KWTMPFILE=$(mktemp)
+${COMPARE_ECL_COMMAND} -t ${TYPE} -l -P ${FILE1} ${FILE2} ${ABS_TOL} ${REL_TOL} &> $KWTMPFILE
+
+cat $KWTMPFILE
+echo ""
+
+analyzeEclFailure
+
+rm -f $KWTMPFILE
diff --git a/tests/analyze_summary_failure.sh b/tests/analyze_summary_failure.sh
new file mode 100755
index 000000000..51ac84c7e
--- /dev/null
+++ b/tests/analyze_summary_failure.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+# This will perform some analysis on a failed summary
+# comparisons in a regression tests.
+
+# Analyze summary test failure.
+# - Print number of failed keywords
+# - Print number of levels each keyword failed at
+# - Print maximum deviations for each keyword
+analyzeSummaryFailure() {
+  lines=`cat $TMPFILE | grep "For keyword"`
+  IFS=$'\n'
+  kwds=""
+  for line in $lines
+  do
+    kwds+="`echo $line | awk -F ' ' '{print $3}'`\n"
+  done
+  unique_kwds=`echo -e $kwds | uniq`
+  kws_failed=`echo -e "$unique_kwds" | wc -l`
+  echo "$kws_failed summary keyword(s) exhibit failures"
+  numsteps=`cat $TMPFILE | grep "Comparing " | grep steps | awk -F ' ' '{print $2}'`
+
+  for kwd in $unique_kwds
+  do
+    lines=`cat $TMPFILE | grep -n "For keyword $kwd$"`
+    echo -e "\t $kwd"
+    echo -e "\t \t Fails for: `cat $TMPFILE | grep -n "$kwd$" | wc -l` / $numsteps steps."
+    abs=0
+    rel=0
+    for line in $lines
+    do
+      ln=`echo $line | awk -F ':' '{print $1'}`
+      abs_line=$(($ln+2))
+      rel_line=$(($ln+3))
+      abs_new=`sed "${abs_line}q;d" $TMPFILE | awk -F ' ' '{print $5}'`
+      rel_new=`sed "${rel_line}q;d" $TMPFILE | awk -F ' ' '{print $5}'`
+      abs_new=`echo ${abs_new: : -1} | awk '{printf sprintf("%.16f", $1); }'`
+      rel_new=`echo ${rel_new: : -1} | awk '{printf sprintf("%.16f", $1); }'`
+      if [ `bc <<< "$abs_new>$abs"` -eq 1 ]
+      then
+        abs=$abs_new
+      fi
+      if [ `bc <<< "$rel_new>$rel"` -eq 1 ]
+      then
+        rel=$rel_new
+      fi
+    done
+    echo -e "\t\t Largest absolute error: `echo $abs | awk '{printf sprintf("%e", $1); }'`"
+    echo -e "\t\t Largest relative error: `echo $rel | awk '{printf sprintf("%e", $1); }'`"
+  done
+}
+
+COMPARE_SUMMARY_COMMAND=$1
+PARAM=$2
+FILE1=$3
+FILE2=$4
+ABS_TOL=$5
+REL_TOL=$6
+
+TMPFILE=$(mktemp)
+${COMPARE_SUMMARY_COMMAND} -p -n ${PARAM} ${FILE1} ${FILE2} ${ABS_TOL} ${REL_TOL} &> $TMPFILE
+
+analyzeSummaryFailure
+rm -f $TMPFILE
diff --git a/tests/run-init-regressionTest.sh b/tests/run-init-regressionTest.sh
index 168f2aaf5..c68e90189 100755
--- a/tests/run-init-regressionTest.sh
+++ b/tests/run-init-regressionTest.sh
@@ -1,5 +1,10 @@
 #!/bin/bash
-set -e
+
+# This runs the initialization step of a simulator,
+# then compares the resulting INIT file against a reference.
+# This is meant to track regressions in INIT file writing.
+# Useful for models that are too large to do simulation on
+# as a regression test.
 
 INPUT_DATA_PATH="$1"
 RESULT_PATH="$2"
@@ -19,4 +24,12 @@ cd ${RESULT_PATH}
 ${BINPATH}/${EXE_NAME} ${TEST_ARGS} nosim=true
 cd ..
 
+ecode=0
 ${COMPARE_ECL_COMMAND} -t INIT ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
+if [ $? -ne 0 ]
+then
+  ecode=1
+  `dirname $0`/analyze_ecl_failure.sh ${COMPARE_ECL_COMMAND} INIT ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
+fi
+
+exit $ecode
diff --git a/tests/run-parallel-regressionTest.sh b/tests/run-parallel-regressionTest.sh
index 7d7a7209e..acbe80402 100755
--- a/tests/run-parallel-regressionTest.sh
+++ b/tests/run-parallel-regressionTest.sh
@@ -1,5 +1,9 @@
 #!/bin/bash
 
+# This performs a serial and a parallel for a simulator,
+# then compares the summary and restart files from the two runs.
+# Meant to track regression in parallel simulators.
+
 INPUT_DATA_PATH="$1"
 RESULT_PATH="$2"
 BINPATH="$3"
@@ -28,8 +32,13 @@ ${COMPARE_SUMMARY_COMMAND} -R ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FI
 if [ $? -ne 0 ]
 then
   ecode=1
+  `dirname $0`/analyze_summary_failure.sh ${COMPARE_SUMMARY_COMMAND} -R ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL}
 fi
 ${COMPARE_ECL_COMMAND} -l ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL}
-test $? -eq 0 || ecode=1
+if [ $? -ne 0 ]
+then
+  ecode=1
+  `dirname $0`/analyze_ecl_failure.sh ${COMPARE_ECL_COMMAND} UNRST ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/mpi/${FILENAME} ${ABS_TOL} ${REL_TOL}
+fi
 
 exit $ecode
diff --git a/tests/run-regressionTest.sh b/tests/run-regressionTest.sh
index ce00f6e75..96d05a5db 100755
--- a/tests/run-regressionTest.sh
+++ b/tests/run-regressionTest.sh
@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# This runs a simulator, then compares the summary, restart and init
+# files against a reference.
+
 INPUT_DATA_PATH="$1"
 RESULT_PATH="$2"
 BINPATH="$3"
@@ -23,12 +26,21 @@ ${COMPARE_SUMMARY_COMMAND} -r ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-
 if [ $? -ne 0 ]
 then
   ecode=1
+  `dirname $0`/analyze_summary_failure.sh ${COMPARE_SUMMARY_COMMAND} -r ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
 fi
 
 ${COMPARE_ECL_COMMAND}  ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
-test $? -eq 0 || ecode=1
+if [ $? -ne 0 ]
+then
+  ecode=1
+  `dirname $0`/analyze_ecl_failure.sh ${COMPARE_ECL_COMMAND} UNRST ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
+fi
 
 ${COMPARE_ECL_COMMAND} -t INIT ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
-test $? -eq 0 || ecode=1
+if [ $? -ne 0 ]
+then
+  ecode=1
+  `dirname $0`/analyze_ecl_failure.sh ${COMPARE_ECL_COMMAND} INIT ${RESULT_PATH}/${FILENAME} ${INPUT_DATA_PATH}/opm-simulation-reference/${FILENAME} ${ABS_TOL} ${REL_TOL}
+fi
 
 exit $ecode
diff --git a/tests/run-restart-regressionTest.sh b/tests/run-restart-regressionTest.sh
index 3fcd5cf38..8fbd5043b 100755
--- a/tests/run-restart-regressionTest.sh
+++ b/tests/run-restart-regressionTest.sh
@@ -1,5 +1,9 @@
 #!/bin/bash
 
+# This runs a simulator from start to end, then a restarted
+# run of the simulator, before comparing the output from the two runs.
+# This is meant to track regressions in the restart support.
+
 INPUT_DATA_PATH="$1"
 RESULT_PATH="$2"
 BINPATH="$3"
@@ -25,9 +29,14 @@ ${COMPARE_SUMMARY_COMMAND} -R ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/${FILENA
 if [ $? -ne 0 ]
 then
   ecode=1
+  `dirname $0`/analyze_summary_failure.sh ${COMPARE_SUMMARY_COMMAND} -R ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/${FILENAME}_RESTART ${ABS_TOL} ${REL_TOL}
 fi
 
 ${COMPARE_ECL_COMMAND} -l ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/${FILENAME}_RESTART ${ABS_TOL} ${REL_TOL}
-test $? -eq 0 || ecode=1
+if [ $? -ne 0 ]
+then
+  ecode=1
+  `dirname $0`/analyze_ecl_failure.sh ${COMPARE_ECL_COMMAND} UNRST ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/${FILENAME}_RESTART ${ABS_TOL} ${REL_TOL}
+fi
 
 exit $ecode