OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5168 from lisajulia/feature/removeFilesOnSimulationStartup

Browse Source 
Feature/remove files on simulation startup
This commit is contained in:
Bård Skaflestad
2024-02-08 17:30:07 +01:00
committed by GitHub
parent 6729190af1 7077cfb35c
commit d69ba021a9
5 changed files with 100 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
opm/simulators/flow/Main.cpp
View File

@@ -188,14 +188,13 @@ void Main::readDeck(const std::string& deckFilename,
const bool init_from_restart_file,
const bool allRanksDbgPrtLog,
const std::string& parsingStrictness,
const int mpiRank,
const std::size_t numThreads,
const int output_param,
const std::string& parameters,
std::string_view moduleVersion,
std::string_view compileTimestamp)
{
auto omode = setupLogging(mpiRank,
auto omode = setupLogging(EclGenericVanguard::comm(),
deckFilename,
outputDir,
outputMode,

2
opm/simulators/flow/Main.hpp
View File

@@ -417,7 +417,6 @@ private:
!EWOMS_GET_PARAM(PreTypeTag, bool, SchedRestart),
EWOMS_GET_PARAM(PreTypeTag, bool, EnableLoggingFalloutWarning),
EWOMS_GET_PARAM(PreTypeTag, std::string, ParsingStrictness),
mpiRank,
getNumThreads<PreTypeTag>(),
EWOMS_GET_PARAM(PreTypeTag, int, EclOutputInterval),
cmdline_params,
@@ -692,7 +691,6 @@ private:
const bool init_from_restart_file,
const bool allRanksDbgPrtLog,
const std::string& parsingStrictness,
const int mpiRank,
const std::size_t numThreads,
const int output_param,
const std::string& parameters,

47
opm/simulators/utils/readDeck.cpp
View File

@@ -76,6 +76,7 @@
#include <filesystem>
#include <functional>
#include <memory>
#include <regex>
#include <sstream>
#include <stdexcept>
#include <unordered_map>
@@ -402,15 +403,28 @@ void Opm::ensureOutputDirExists(const std::string& cmdline_output_dir)
}
}
void Opm::prepareResultOutputDirectory(const std::string& baseName,
const std::filesystem::path& outputDir)
{
//Loop over all files in the output directory and subdirectories and delete them if their name is baseName + a correct extension
std::regex r(baseName + R"(\.(F?(DBG|E?GRID|INIT|PRT|RFT|SMSPEC|UNSMRY|UNRST)|([ABCFGHSTUXYZ]\d{4})|(INFOSTEP|INFOITER|OPMRST)))");
for (auto& file : std::filesystem::recursive_directory_iterator(outputDir)) {
std::string fileName = file.path().filename();
if (std::regex_match(fileName, r)) {
std::filesystem::remove(file);
}
}
}
// Setup the OpmLog backends
Opm::FileOutputMode
Opm::setupLogging(const int mpi_rank_,
const std::string& deck_filename,
const std::string& cmdline_output_dir,
const std::string& cmdline_output,
const bool output_cout_,
const std::string& stdout_log_id,
const bool allRanksDbgLog)
Opm::setupLogging(Parallel::Communication& comm,
const std::string& deck_filename,
const std::string& cmdline_output_dir,
const std::string& cmdline_output,
const bool output_cout_,
const std::string& stdout_log_id,
const bool allRanksDbgLog)
{
if (!cmdline_output_dir.empty()) {
ensureOutputDirExists(cmdline_output_dir);
@@ -439,15 +453,24 @@ Opm::setupLogging(const int mpi_rank_,
: std::filesystem::current_path().generic_string();
}
// Cleans up the result output directory, we only do this on process 0
if (comm.rank() == 0) {
prepareResultOutputDirectory(baseName, output_dir);
}
//... and the other processes need to wait for this to be finished.
#if HAVE_MPI
MPI_Barrier(comm);
#endif
logFileStream << output_dir << "/" << baseName;
debugFileStream << output_dir << "/" << baseName;
if (mpi_rank_ != 0) {
if (comm.rank() != 0) {
// Added rank to log file for non-zero ranks.
// This prevents message loss.
debugFileStream << "." << mpi_rank_;
debugFileStream << "." << comm.rank();
// If the following file appears then there is a bug.
logFileStream << "." << mpi_rank_;
logFileStream << "." << comm.rank();
}
logFileStream << ".PRT";
debugFileStream << ".DBG";
@@ -469,7 +492,7 @@ Opm::setupLogging(const int mpi_rank_,
std::cerr << "Value " << cmdline_output
<< " is not a recognized output mode. Using \"all\" instead.\n";
}
if (!allRanksDbgLog && mpi_rank_ != 0)
if (!allRanksDbgLog && comm.rank() != 0)
{
output = FileOutputMode::OUTPUT_NONE;
}
@@ -488,7 +511,7 @@ Opm::setupLogging(const int mpi_rank_,
Opm::OpmLog::addBackend("DEBUGLOG", debugLog);
}
if (mpi_rank_ == 0) {
if (comm.rank() == 0) {
std::shared_ptr<Opm::StreamLog> streamLog = std::make_shared<Opm::StreamLog>(std::cout, Opm::Log::StdoutMessageTypes);
Opm::OpmLog::addBackend(stdout_log_id, streamLog);
// Set a tag limit of 10 (no category limit). Will later in

19
opm/simulators/utils/readDeck.hpp
View File

@@ -24,6 +24,7 @@
#include <opm/simulators/utils/ParallelCommunication.hpp>
#include <filesystem>
#include <memory>
#include <optional>
#include <string>
@@ -60,17 +61,21 @@ enum class FileOutputMode {
void
ensureOutputDirExists(const std::string& cmdline_output_dir);
// Prepare the result ouptut directory by removing files from previous simulation runs.
void
prepareResultOutputDirectory(const std::string& baseName, const std::filesystem::path& outputDir);
std::unique_ptr<ParseContext> setupParseContext(const bool exitOnAllErrors);
// Setup the OpmLog backends
FileOutputMode
setupLogging(int mpi_rank_,
const std::string& deck_filename,
const std::string& cmdline_output_dir,
const std::string& cmdline_output,
bool output_cout_,
const std::string& stdout_log_id,
const bool allRanksDbgLog);
setupLogging(Parallel::Communication& comm,
const std::string& deck_filename,
const std::string& cmdline_output_dir,
const std::string& cmdline_output,
bool output_cout_,
const std::string& stdout_log_id,
const bool allRanksDbgLog);
/// \brief Reads the deck and creates all necessary objects if needed
///

71
tests/test_outputdir.cpp
View File

@@ -95,27 +95,46 @@ BOOST_FIXTURE_TEST_CASE(WithOutputDir, Fixture)
using PathPair = std::pair<std::filesystem::path, std::filesystem::path>;
for (const auto& Case : {PathPair{input_path, input_path / "output1"},
PathPair{input_path / "subdir", input_path / "output2"},
PathPair{input_path / "subdir" / "subdir", input_path / "output3"}}) {
const std::string output_path = "--output-dir=" + Case.second.string();
const std::string input_file_path = (Case.first / "INPUT.DATA");
const char* no_param[] = {"test_outputdir", input_file_path.c_str(),
output_path.c_str(), nullptr};
for (const bool createFaultyFileWithDirectory : {true, false}) {
for (const auto& Case : {PathPair{input_path, input_path / "output1"},
PathPair{input_path / "subdir", input_path / "output2"},
PathPair{input_path / "subdir" / "subdir", input_path / "output3"}}) {
const std::string output_path = "--output-dir=" + Case.second.string();
const std::string input_file_path = (Case.first / "INPUT.DATA");
using ParamsMeta = Opm::GetProp<Opm::Properties::TTag::FlowEarlyBird,
Opm::Properties::ParameterMetaData>;
ParamsMeta::clear();
const std::string output_dbg_path = (Case.second / "INPUT.DBG");
if (createFaultyFileWithDirectory) {
std::filesystem::create_directories(Case.second);
// Create file with faulty content
std::string dummy = R"(dummy)";
std::ofstream of(output_dbg_path);
of << dummy << std::endl;
}
Opm::Main main(3, const_cast<char**>(no_param), false);
const char* no_param[] = {"test_outputdir", input_file_path.c_str(),
output_path.c_str(), nullptr};
BOOST_CHECK_EQUAL(main.justInitialize(), EXIT_SUCCESS);
BOOST_CHECK(!std::filesystem::exists(input_path / "INPUT.PRT"));
BOOST_CHECK(!std::filesystem::exists(input_path / "INPUT.DBG"));
BOOST_CHECK(!std::filesystem::exists(Case.first / "INPUT.PRT"));
BOOST_CHECK(!std::filesystem::exists(Case.first / "INPUT.DBG"));
BOOST_CHECK(std::filesystem::exists(Case.second / "INPUT.PRT"));
BOOST_CHECK(std::filesystem::exists(Case.second / "INPUT.DBG"));
using ParamsMeta = Opm::GetProp<Opm::Properties::TTag::FlowEarlyBird,
Opm::Properties::ParameterMetaData>;
ParamsMeta::clear();
Opm::Main main(3, const_cast<char**>(no_param), false);
BOOST_CHECK_EQUAL(main.justInitialize(), EXIT_SUCCESS);
// Check if the files have been created at the right spot
BOOST_CHECK(!std::filesystem::exists(input_path / "INPUT.PRT"));
BOOST_CHECK(!std::filesystem::exists(input_path / "INPUT.DBG"));
BOOST_CHECK(!std::filesystem::exists(Case.first / "INPUT.PRT"));
BOOST_CHECK(!std::filesystem::exists(Case.first / "INPUT.DBG"));
BOOST_CHECK(std::filesystem::exists(Case.second / "INPUT.PRT"));
BOOST_CHECK(std::filesystem::exists(output_dbg_path));
// Check if the file with the faulty content has been removed and replaced by a valid one
std::ifstream file(output_dbg_path);
std::string line;
std::getline(file, line);
BOOST_CHECK(line.find("dummy") == std::string::npos);
}
}
}
@@ -126,6 +145,13 @@ BOOST_FIXTURE_TEST_CASE(NoOutputDir, Fixture)
for (const auto& Case : {input_path / "subdir" / "subdir",
input_path / "subdir"}) {
const std::string input_file_path = (Case / "INPUT.DATA");
const std::string output_dbg_path = (Case / "INPUT.DBG");
// Create file with faulty content
std::string dummy = R"(dummy)";
std::ofstream of(output_dbg_path);
of << dummy << std::endl;
const char* no_param[] = {"test_outputdir", input_file_path.c_str(), nullptr};
using ParamsMeta = Opm::GetProp<Opm::Properties::TTag::FlowEarlyBird,
@@ -135,10 +161,17 @@ BOOST_FIXTURE_TEST_CASE(NoOutputDir, Fixture)
Opm::Main main(2, const_cast<char**>(no_param), false);
BOOST_CHECK_EQUAL(main.justInitialize(), EXIT_SUCCESS);
// Check if the files have been created at the right spot
BOOST_CHECK(!std::filesystem::exists(input_path / "INPUT.PRT"));
BOOST_CHECK(!std::filesystem::exists(input_path / "INPUT.DBG"));
BOOST_CHECK(std::filesystem::exists(Case/ "INPUT.PRT"));
BOOST_CHECK(std::filesystem::exists(Case/ "INPUT.DBG"));
BOOST_CHECK(std::filesystem::exists(output_dbg_path));
// Check if the file with the faulty content has been removed and replaced by a valid one
std::ifstream file(output_dbg_path);
std::string line;
std::getline(file, line);
BOOST_CHECK(line.find("dummy") == std::string::npos);
}
}