From ccac548420319ff9a8d5b9df1b232cf17e35a653 Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Tue, 8 Sep 2015 09:46:35 +0200 Subject: [PATCH 1/7] Introduces parallel exception handling for ILU0 preconditioner. When running Norne with the interleaved solver sometimes exceptions (diagonal matrix block is not invertible) occur for some rows in the ILU decomposition. In a parallel run this means that some, but not all processes will see the exceptions. This leads to a classic deadlock. With this commit we catch the exception during the setup of the preconditioner, determine with the other processes whether there were any exceptions, and in this case all the processes will throw an exception. Currently this limited to Dune::MatrixBlockError, but we could extend this. --- .../NewtonIterationBlackoilInterleaved.hpp | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp b/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp index 4eef1944e..1b3b7c43f 100644 --- a/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp +++ b/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp @@ -132,13 +132,28 @@ namespace Opm AdditionalObjectDeleter > constructPrecond(Operator& opA, const Comm& comm) const { - const double relax = 1.0; - SeqPreconditioner* seq_precond= new SeqPreconditioner(opA.getmat(), - relax); typedef AdditionalObjectDeleter Deleter; - std::unique_ptr - precond(new ParPreconditioner(*seq_precond, comm), - Deleter(*seq_precond)); + typedef std::unique_ptr Pointer; + Pointer precond; + int ilu_setup_successful = 1; + std::string message; + const double relax = 1.0; + try{ + SeqPreconditioner* seq_precond= new SeqPreconditioner(opA.getmat(), + relax); + precond = Pointer(new ParPreconditioner(*seq_precond, comm), + Deleter(*seq_precond)); + }catch(Dune::MatrixBlockError error) + { + message = error.what(); + std::cerr< Date: Tue, 8 Sep 2015 11:15:46 +0200 Subject: [PATCH 2/7] Put space around braces where it seems approppriate. --- opm/autodiff/NewtonIterationBlackoilInterleaved.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp b/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp index 1b3b7c43f..bb85d6202 100644 --- a/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp +++ b/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp @@ -138,19 +138,20 @@ namespace Opm int ilu_setup_successful = 1; std::string message; const double relax = 1.0; - try{ + try { SeqPreconditioner* seq_precond= new SeqPreconditioner(opA.getmat(), relax); precond = Pointer(new ParPreconditioner(*seq_precond, comm), Deleter(*seq_precond)); - }catch(Dune::MatrixBlockError error) + } + catch ( Dune::MatrixBlockError error ) { message = error.what(); std::cerr< Date: Tue, 8 Sep 2015 11:44:46 +0200 Subject: [PATCH 3/7] Fix typo wether->whether. --- opm/autodiff/NewtonIterationBlackoilInterleaved.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp b/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp index bb85d6202..7696b25d8 100644 --- a/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp +++ b/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp @@ -150,7 +150,7 @@ namespace Opm std::cerr< Date: Tue, 8 Sep 2015 11:48:06 +0200 Subject: [PATCH 4/7] Include rank into error message. --- opm/autodiff/NewtonIterationBlackoilInterleaved.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp b/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp index 7696b25d8..89ca5ec64 100644 --- a/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp +++ b/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp @@ -147,7 +147,10 @@ namespace Opm catch ( Dune::MatrixBlockError error ) { message = error.what(); - std::cerr< Date: Tue, 8 Sep 2015 11:54:45 +0200 Subject: [PATCH 5/7] Catch and process exception during ILU0 in parallel for CPR, too. --- opm/autodiff/CPRPreconditioner.hpp | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/opm/autodiff/CPRPreconditioner.hpp b/opm/autodiff/CPRPreconditioner.hpp index 1dbd627db..f4ce2300a 100644 --- a/opm/autodiff/CPRPreconditioner.hpp +++ b/opm/autodiff/CPRPreconditioner.hpp @@ -178,7 +178,26 @@ createILU0Ptr(const M& A, double relax, Dune::OwnerOverlapCopyCommunication, Dune::SeqILU0 > PointerType; - Dune::SeqILU0* ilu = new Dune::SeqILU0(A, relax); + Dune::SeqILU0* ilu = nullptr; + int ilu_setup_successful = 1; + std::string message; + try { + ilu = new Dune::SeqILU0(A, relax); + } + catch ( Dune::MatrixBlockError error ) + { + message = error.what(); + std::cerr<<"Exception occured on process " << + comm.communicator().rank() << " during " << + "setup of ILU0 preconditioner with message: " << + message<, I1, I2> ::type ( new PointerType(*ilu, comm), createParallelDeleter(*ilu, comm)); } From 3c1498ce923a3187148f551ac97523610071cf9c Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Tue, 8 Sep 2015 12:06:03 +0200 Subject: [PATCH 6/7] Only construct SeqILU0 in try block. The rest is not needed and this simplyfies code. --- opm/autodiff/NewtonIterationBlackoilInterleaved.hpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp b/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp index 89ca5ec64..470915135 100644 --- a/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp +++ b/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp @@ -134,15 +134,13 @@ namespace Opm { typedef AdditionalObjectDeleter Deleter; typedef std::unique_ptr Pointer; - Pointer precond; int ilu_setup_successful = 1; std::string message; const double relax = 1.0; + SeqPreconditioner* seq_precond = nullptr; try { - SeqPreconditioner* seq_precond= new SeqPreconditioner(opA.getmat(), - relax); - precond = Pointer(new ParPreconditioner(*seq_precond, comm), - Deleter(*seq_precond)); + seq_precond= new SeqPreconditioner(opA.getmat(), + relax); } catch ( Dune::MatrixBlockError error ) { @@ -158,7 +156,8 @@ namespace Opm { throw Dune::MatrixBlockError(); } - return precond; + return Pointer(new ParPreconditioner(*seq_precond, comm), + Deleter(*seq_precond)); } #endif From 32b8ed654ecc664d06d74a3d540e2e0bd2f83dfb Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Tue, 8 Sep 2015 14:38:29 +0200 Subject: [PATCH 7/7] Prevent memory leaks in case SeqILU0 throws an exception. --- opm/autodiff/CPRPreconditioner.hpp | 5 +++++ opm/autodiff/NewtonIterationBlackoilInterleaved.hpp | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/opm/autodiff/CPRPreconditioner.hpp b/opm/autodiff/CPRPreconditioner.hpp index f4ce2300a..c30b6e1b1 100644 --- a/opm/autodiff/CPRPreconditioner.hpp +++ b/opm/autodiff/CPRPreconditioner.hpp @@ -196,6 +196,11 @@ createILU0Ptr(const M& A, double relax, // Check whether there was a problem on some process if ( comm.communicator().min(ilu_setup_successful) == 0 ) { + if ( ilu ) // not null if constructor succeeded + { + // prevent memory leak + delete ilu; + } throw Dune::MatrixBlockError(); } return typename SelectParallelILUSharedPtr, I1, I2> diff --git a/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp b/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp index 470915135..2a8553c64 100644 --- a/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp +++ b/opm/autodiff/NewtonIterationBlackoilInterleaved.hpp @@ -139,7 +139,7 @@ namespace Opm const double relax = 1.0; SeqPreconditioner* seq_precond = nullptr; try { - seq_precond= new SeqPreconditioner(opA.getmat(), + seq_precond = new SeqPreconditioner(opA.getmat(), relax); } catch ( Dune::MatrixBlockError error ) @@ -154,6 +154,11 @@ namespace Opm // Check whether there was a problem on some process if ( comm.communicator().min(ilu_setup_successful) == 0 ) { + if ( seq_precond ) // not null if constructor succeeded + { + // prevent memory leak + delete seq_precond; + } throw Dune::MatrixBlockError(); } return Pointer(new ParPreconditioner(*seq_precond, comm),