From 52270e7ac79f1fb7e426db8ca2d7f1b1d9891d17 Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Mon, 14 Jan 2019 12:48:18 +0100 Subject: [PATCH 1/6] Use uppercase basename for *DBG and *PRT files for any extension. Previously runs of flow started with 'flow base.' Would result in base..DBG files; note the double dot and the case sensitivity. Now the file will be BASE.DBG in this case. Note that any extension will be stipped (not only .DATA as before). --- opm/autodiff/FlowMainEbos.hpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/opm/autodiff/FlowMainEbos.hpp b/opm/autodiff/FlowMainEbos.hpp index 0f2ce66a7..33e098d16 100755 --- a/opm/autodiff/FlowMainEbos.hpp +++ b/opm/autodiff/FlowMainEbos.hpp @@ -321,11 +321,8 @@ namespace Opm std::ostringstream debugFileStream; std::ostringstream logFileStream; - if (boost::to_upper_copy(path(fpath.extension()).string()) == ".DATA") { - baseName = path(fpath.stem()).string(); - } else { - baseName = path(fpath.filename()).string(); - } + // Strip extension if any + baseName = boost::to_upper_copy(path(fpath.stem()).string()); const std::string& output_dir = eclState().getIOConfig().getOutputDir(); logFileStream << output_dir << "/" << baseName; @@ -425,9 +422,10 @@ namespace Opm const std::string& output_dir = eclState().getIOConfig().getOutputDir(); fs::path output_path(output_dir); fs::path deck_filename(EWOMS_GET_PARAM(TypeTag, std::string, EclDeckFileName)); + std::string basename = boost::to_upper_copy(deck_filename.stem().string()); std::for_each(fs::directory_iterator(output_path), fs::directory_iterator(), - detail::ParallelFileMerger(output_path, deck_filename.stem().string())); + detail::ParallelFileMerger(output_path, basename)); } void setupEbosSimulator() From 4b6144bb65d2b4ea46bcfa4b592ea7f1f5a7c7ba Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Mon, 14 Jan 2019 13:06:21 +0100 Subject: [PATCH 2/6] Only warn and show parallel logging fallout on demand. This commit adds an option that allows to enable detecting parallel logging fallout (option --enable-logging-fallout-warning). It is now false by default (previous behavior was as if the option was true). If option is true a warning will be printed for any process with nonzero rank that does try to log to *.PRT or *.DBG and the logged output will be appended to these files at the end. --- opm/autodiff/FlowMainEbos.hpp | 12 +++++-- opm/simulators/ParallelFileMerger.hpp | 51 ++++++++++++++++++--------- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/opm/autodiff/FlowMainEbos.hpp b/opm/autodiff/FlowMainEbos.hpp index 33e098d16..ddce566a7 100755 --- a/opm/autodiff/FlowMainEbos.hpp +++ b/opm/autodiff/FlowMainEbos.hpp @@ -58,12 +58,14 @@ NEW_PROP_TAG(OutputMode); NEW_PROP_TAG(EnableDryRun); NEW_PROP_TAG(OutputInterval); NEW_PROP_TAG(UseAmg); +NEW_PROP_TAG(EnableLoggingFalloutWarning); SET_STRING_PROP(EclFlowProblem, OutputMode, "all"); // TODO: enumeration parameters. we use strings for now. SET_STRING_PROP(EclFlowProblem, EnableDryRun, "auto"); - +// Do not merge parallel output files or warn about them +SET_BOOL_PROP(EclFlowProblem, EnableLoggingFalloutWarning, false); SET_INT_PROP(EclFlowProblem, OutputInterval, 1); END_PROPERTIES @@ -106,6 +108,9 @@ namespace Opm "Specify if the simulation ought to be actually run, or just pretended to be"); EWOMS_REGISTER_PARAM(TypeTag, int, OutputInterval, "Specify the number of report steps between two consecutive writes of restart data"); + EWOMS_REGISTER_PARAM(TypeTag, bool, EnableLoggingFalloutWarning, + "Developer option to see whether logging was on non-root processors. In that case it will be appended to the *.DBG or *.PRT files"); + Simulator::registerParameters(); ISTLSolverType::registerParameters(); @@ -422,10 +427,11 @@ namespace Opm const std::string& output_dir = eclState().getIOConfig().getOutputDir(); fs::path output_path(output_dir); fs::path deck_filename(EWOMS_GET_PARAM(TypeTag, std::string, EclDeckFileName)); - std::string basename = boost::to_upper_copy(deck_filename.stem().string()); + std::string basename = boost::to_upper_copy(fs::path(deck_filename).stem().string()); std::for_each(fs::directory_iterator(output_path), fs::directory_iterator(), - detail::ParallelFileMerger(output_path, basename)); + detail::ParallelFileMerger(output_path, basename, + EWOMS_GET_PARAM(TypeTag, bool, EnableLoggingFalloutWarning)); } void setupEbosSimulator() diff --git a/opm/simulators/ParallelFileMerger.hpp b/opm/simulators/ParallelFileMerger.hpp index ee237c503..37590af33 100644 --- a/opm/simulators/ParallelFileMerger.hpp +++ b/opm/simulators/ParallelFileMerger.hpp @@ -48,19 +48,25 @@ public: /// \param output_dir The output directory to use for reading/Writing. /// \param deckanme The name of the deck. ParallelFileMerger(const fs::path& output_dir, - const std::string& deckname) + const std::string& deckname, + bool show_fallout = false) : debugFileRegex_(deckname+"\\.\\d+\\.DBG"), logFileRegex_(deckname+"\\.\\d+\\.PRT"), - fileWarningRegex_(deckname+"\\.(\\d+)\\.[^.]+") + fileWarningRegex_(deckname+"\\.(\\d+)\\.[^.]+"), + show_fallout_(show_fallout) { - auto debugPath = output_dir; - debugPath /= (deckname + ".DBG"); - debugStream_.reset(new fs::ofstream(debugPath, - std::ofstream::app)); - auto logPath = output_dir; - logPath /= ( deckname + ".PRT"); - logStream_.reset(new fs::ofstream(logPath, - std::ofstream::app)); + std::cout<<"show_fallout="< debugStream_; /// \brief Stream to *.PRT file std::unique_ptr logStream_; + /// \brief Whether to show any logging fallout + bool show_fallout_; }; } // end namespace detail } // end namespace OPM From 0c2edcfe4741184a676624c1da90184a9455ec92 Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Wed, 16 Jan 2019 11:56:47 +0100 Subject: [PATCH 3/6] Remove debugging output --- opm/simulators/ParallelFileMerger.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/opm/simulators/ParallelFileMerger.hpp b/opm/simulators/ParallelFileMerger.hpp index 37590af33..e8f0e83d5 100644 --- a/opm/simulators/ParallelFileMerger.hpp +++ b/opm/simulators/ParallelFileMerger.hpp @@ -55,7 +55,6 @@ public: fileWarningRegex_(deckname+"\\.(\\d+)\\.[^.]+"), show_fallout_(show_fallout) { - std::cout<<"show_fallout="< Date: Wed, 16 Jan 2019 11:57:28 +0100 Subject: [PATCH 4/6] [bugfix] Use file not filename to remove files not in current directory. Passing just the filename to remove only will remove files in the current directory. When --output-dir the files would not have been removed. --- opm/simulators/ParallelFileMerger.hpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/opm/simulators/ParallelFileMerger.hpp b/opm/simulators/ParallelFileMerger.hpp index e8f0e83d5..10eacf306 100644 --- a/opm/simulators/ParallelFileMerger.hpp +++ b/opm/simulators/ParallelFileMerger.hpp @@ -82,7 +82,7 @@ public: if ( show_fallout_ ){ appendFile(*logStream_, file, rank); }else{ - fs::remove(filename); + fs::remove(file); } } else @@ -91,9 +91,8 @@ public: { if ( show_fallout_ ){ appendFile(*debugStream_, file, rank); - }else - { - fs::remove(filename); + }else{ + fs::remove(file); } } else From 4f4f5cbca3206fa01712da72f3d0aedf216cee57 Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Wed, 16 Jan 2019 11:58:47 +0100 Subject: [PATCH 5/6] [bugfix] Only strip ".DATA" or "." extensions. As there might models named MODEL.01.DATA for which a call "flow MODEL.01" would have stripped the extension. --- opm/autodiff/FlowMainEbos.hpp | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/opm/autodiff/FlowMainEbos.hpp b/opm/autodiff/FlowMainEbos.hpp index ddce566a7..04fdaee66 100755 --- a/opm/autodiff/FlowMainEbos.hpp +++ b/opm/autodiff/FlowMainEbos.hpp @@ -326,8 +326,16 @@ namespace Opm std::ostringstream debugFileStream; std::ostringstream logFileStream; - // Strip extension if any - baseName = boost::to_upper_copy(path(fpath.stem()).string()); + // Strip extension "." or ".DATA" + std::string extension = boost::to_upper_copy(fpath.extension().string()); + if ( extension == ".DATA" || extension == "." ) + { + baseName = boost::to_upper_copy(fpath.stem().string()); + } + else + { + baseName = boost::to_upper_copy(fpath.filename().string()); + } const std::string& output_dir = eclState().getIOConfig().getOutputDir(); logFileStream << output_dir << "/" << baseName; @@ -427,7 +435,17 @@ namespace Opm const std::string& output_dir = eclState().getIOConfig().getOutputDir(); fs::path output_path(output_dir); fs::path deck_filename(EWOMS_GET_PARAM(TypeTag, std::string, EclDeckFileName)); - std::string basename = boost::to_upper_copy(fs::path(deck_filename).stem().string()); + std::string basename; + // Strip extension "." and ".DATA" + std::string extension = boost::to_upper_copy(deck_filename.extension().string()); + if ( extension == ".DATA" || extension == "." ) + { + basename = boost::to_upper_copy(deck_filename.stem().string()); + } + else + { + basename = boost::to_upper_copy(deck_filename.filename().string()); + } std::for_each(fs::directory_iterator(output_path), fs::directory_iterator(), detail::ParallelFileMerger(output_path, basename, From 2dd0b62effb7de59f004d0cf462b5ad985107d0d Mon Sep 17 00:00:00 2001 From: Markus Blatt Date: Wed, 16 Jan 2019 12:01:34 +0100 Subject: [PATCH 6/6] [bugfix] Added missing ) to make code compile --- opm/autodiff/FlowMainEbos.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opm/autodiff/FlowMainEbos.hpp b/opm/autodiff/FlowMainEbos.hpp index 04fdaee66..b9f2c759c 100755 --- a/opm/autodiff/FlowMainEbos.hpp +++ b/opm/autodiff/FlowMainEbos.hpp @@ -449,7 +449,7 @@ namespace Opm std::for_each(fs::directory_iterator(output_path), fs::directory_iterator(), detail::ParallelFileMerger(output_path, basename, - EWOMS_GET_PARAM(TypeTag, bool, EnableLoggingFalloutWarning)); + EWOMS_GET_PARAM(TypeTag, bool, EnableLoggingFalloutWarning))); } void setupEbosSimulator()