From bb38c57b1b5f2cffa2cee084b44f0dc31641f015 Mon Sep 17 00:00:00 2001 From: josh bowden Date: Sat, 23 Sep 2023 22:24:11 +0200 Subject: [PATCH] Update for support of Damaris mesh based in situ processing Moved damaris command line parameter accessors to Main.cpp and fixed support for both Python and Paraview Python scripts, as they both cannot be present in the same simulation (seems to be an initialization conflict or double initialization) --- opm/simulators/flow/Main.cpp | 38 ++++++- opm/simulators/flow/Main.hpp | 42 +------- opm/simulators/utils/DamarisKeywords.cpp | 104 +++++++++++++++++--- opm/simulators/utils/DamarisKeywords.hpp | 11 +++ opm/simulators/utils/initDamarisXmlFile.cpp | 8 +- 5 files changed, 144 insertions(+), 59 deletions(-) diff --git a/opm/simulators/flow/Main.cpp b/opm/simulators/flow/Main.cpp index f29ed5d48..04965a442 100644 --- a/opm/simulators/flow/Main.cpp +++ b/opm/simulators/flow/Main.cpp @@ -233,12 +233,48 @@ void Main::setupVanguard() } #if HAVE_DAMARIS -void Main::setupDamaris(const std::string& outputDir , std::map& find_replace_map) +void Main::setupDamaris(const std::string& outputDir ) { + typedef Properties::TTag::FlowEarlyBird PreTypeTag; if (!outputDir.empty()) { ensureOutputDirExists(outputDir); } + bool enableDamarisOutputCollective = true ; + bool saveToDamarisHDF5 = true ; + std::string damarisPythonFilename = "" ; + std::string damarisPythonParaviewFilename = "" ; + + std::string damarisSimName = "" ; // empty defaults to opm-sim- + std::string damarisLogLevel = "info" ; + int nDamarisCores = 1 ; + int nDamarisNodes = 0 ; + long shmemSizeBytes = 536870912 ; + + enableDamarisOutputCollective = EWOMS_GET_PARAM(PreTypeTag, bool, EnableDamarisOutputCollective) ; + saveToDamarisHDF5 = EWOMS_GET_PARAM(PreTypeTag, bool, DamarisSaveToHdf); + damarisPythonFilename = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisPythonScript); + damarisPythonParaviewFilename = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisPythonParaviewScript); + damarisSimName = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisSimName); + nDamarisCores = EWOMS_GET_PARAM(PreTypeTag, int, DamarisDedicatedCores); + nDamarisNodes = EWOMS_GET_PARAM(PreTypeTag, int, DamarisDedicatedNodes); + shmemSizeBytes = EWOMS_GET_PARAM(PreTypeTag, long, DamarisSharedMemeorySizeBytes); + damarisLogLevel = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisLogLevel); + + std::map find_replace_map ; + find_replace_map = Opm::DamarisOutput::DamarisKeywords(EclGenericVanguard::comm(), + outputDir, + enableDamarisOutputCollective, + saveToDamarisHDF5, + nDamarisCores, + nDamarisNodes, + shmemSizeBytes, + damarisPythonFilename, + damarisSimName, + damarisLogLevel, + damarisPythonParaviewFilename + ); + // By default EnableDamarisOutputCollective is true so all simulation results will // be written into one single file for each iteration using Parallel HDF5. // It set to false, FilePerCore mode is used in Damaris, then simulation results in each diff --git a/opm/simulators/flow/Main.hpp b/opm/simulators/flow/Main.hpp index 333117790..561d75910 100644 --- a/opm/simulators/flow/Main.hpp +++ b/opm/simulators/flow/Main.hpp @@ -342,34 +342,9 @@ private: OpmLog::warning(msg); enableDamarisOutput_ = false ; } - + if (enableDamarisOutput_) { - enableDamarisOutputCollective_ = EWOMS_GET_PARAM(PreTypeTag, bool, EnableDamarisOutputCollective) ; - saveToDamarisHDF5_ = EWOMS_GET_PARAM(PreTypeTag, bool, DamarisSaveToHdf); - damarisPythonFilename_ = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisPythonScript); - damarisPythonParaviewFilename_ = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisPythonParaviewScript); - damarisSimName_ = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisSimName); - - nDamarisCores_ = EWOMS_GET_PARAM(PreTypeTag, int, DamarisDedicatedCores); - nDamarisNodes_ = EWOMS_GET_PARAM(PreTypeTag, int, DamarisDedicatedNodes); - shmemSizeBytes_ = EWOMS_GET_PARAM(PreTypeTag, long, DamarisSharedMemeorySizeBytes); - - damarisLogLevel_ = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisLogLevel); - - std::map find_replace_map ; - find_replace_map = Opm::DamarisOutput::DamarisKeywords(EclGenericVanguard::comm(), - outputDir, - enableDamarisOutputCollective_, - saveToDamarisHDF5_, - nDamarisCores_, - nDamarisNodes_, - shmemSizeBytes_, - damarisPythonFilename_, - damarisSimName_, - damarisLogLevel_, - damarisPythonParaviewFilename_ - ); - this->setupDamaris(outputDir, find_replace_map); + this->setupDamaris(outputDir); } #endif // HAVE_DAMARIS @@ -737,8 +712,7 @@ private: } #if HAVE_DAMARIS - void setupDamaris(const std::string& outputDir, - std::map& find_replace_map); + void setupDamaris(const std::string& outputDir); #endif int argc_{0}; @@ -763,16 +737,6 @@ private: bool isSimulationRank_ = true; #if HAVE_DAMARIS bool enableDamarisOutput_ = false; - bool enableDamarisOutputCollective_ = true ; - bool saveToDamarisHDF5_ = true ; - std::string damarisPythonFilename_ = "" ; - std::string damarisPythonParaviewFilename_ = "" ; - - std::string damarisSimName_ = "" ; // empty defaults to opm-sim- - std::string damarisLogLevel_ = "info" ; - int nDamarisCores_ = 1 ; - int nDamarisNodes_ = 0 ; - long shmemSizeBytes_ = 536870912 ; #endif }; diff --git a/opm/simulators/utils/DamarisKeywords.cpp b/opm/simulators/utils/DamarisKeywords.cpp index 1564fb540..88a1d2a06 100644 --- a/opm/simulators/utils/DamarisKeywords.cpp +++ b/opm/simulators/utils/DamarisKeywords.cpp @@ -18,13 +18,16 @@ */ #include +#include #include #include -#include +#include +#include + #include /* - Below is the Damaris Keywords supported by Damaris to be filled + Below are the Damaris Keywords supported by Damaris to be filled in the built-in XML file. The entries in the map below will be filled by the corresponding @@ -34,6 +37,41 @@ namespace Opm::DamarisOutput { + +bool FileExists(const std::string filename_in, const MPI_Comm comm, const int rank) { + // From c++17 : std::filesystem::exists(filename_in); + + int retint = 0 ; + std::ifstream filestr ; + bool file_exists = false ; + + if ((filename_in.length() == 0) || (filename_in == "#") ) { + return file_exists ; + } + + if (rank == 0) { + + filestr.open(filename_in); + file_exists = true ; + if(filestr.fail()) { + retint = 0 ; + } else { + retint = 1 ; + filestr.close() ; + } + MPI_Bcast(&retint,1,MPI_INT,0,comm); + } else { + MPI_Bcast(&retint,1,MPI_INT,0,comm); // recieve the value from rank 0 + } + + if (retint == 1) { + file_exists = true ; + } + + return (file_exists) ; +} + + std::map DamarisKeywords(MPI_Comm comm, std::string OutputDir, bool enableDamarisOutputCollective, @@ -46,11 +84,53 @@ DamarisKeywords(MPI_Comm comm, std::string OutputDir, std::string logLevel, std::string paraviewPythonFilename ) { + int rank ; + MPI_Comm_rank(comm, &rank) ; std::string saveToHDF5_str("MyStore") ; - if (! saveToHDF5 ) saveToHDF5_str = "" ; + if (! saveToHDF5 ) saveToHDF5_str = "#" ; + + std::string disablePythonXMLstart("!--") ; + std::string disablePythonXMLfin("--") ; + std::string disableParaviewXMLstart("!--") ; + std::string disableParaviewXMLfin("--") ; + + + // Test if input Python file exists and set the name of the script for )XML elements std::string publishToPython_str("") ; - if (pythonFilename != "") publishToPython_str="PythonScript" ; // the name of the PyScript XML element + if (pythonFilename != ""){ + if (FileExists(pythonFilename, comm, rank)) { + publishToPython_str="PythonScript" ; // the name of the PyScript XML element + disablePythonXMLstart = std::string("") ; + disablePythonXMLfin = std::string("") ; + } else { + pythonFilename = "" ; // set to empty if it does not exist + std::string disablePythonXMLstart("!--") ; + std::string disablePythonXMLfin("--") ; + } + } + + // Test if input Paraview Python file exists + if (paraviewPythonFilename != ""){ + if (FileExists(paraviewPythonFilename, comm, rank)) { + disableParaviewXMLstart = std::string("") ; + disableParaviewXMLfin = std::string("") ; + } else { + paraviewPythonFilename = "" ; // set to empty if it does not exist + disableParaviewXMLstart = std::string("!--") ; + disableParaviewXMLfin = std::string("--") ; + } + } + + // Flag error if both scripts are enabled + // It would be good to know if Damaris has either one compiled into the library - this is currently not possible + if ((pythonFilename.size() > 0) && (paraviewPythonFilename.size() > 0) ) + { + std::cerr << "ERROR: Both the Python (--damaris-python-script command line argument) and Paraview Python " << + "(--damaris-python-paraview-script command line argument) scripts are valid, however only one type " + "of analysis is supported in a single simulation. Please choose one. Exiting." << std::endl ; + std::exit(-1) ; + } std::string damarisOutputCollective_str("") ; if (enableDamarisOutputCollective) { @@ -64,17 +144,7 @@ DamarisKeywords(MPI_Comm comm, std::string OutputDir, // Having a different simulation name is important if multiple simulations // are running on the same node, as it is used to name the simulations shmem area // and when one sim finishes it removes its shmem file. - // simName_str = "opm-sim-" + damaris::Environment::GetMagicNumber(comm) ; - - // We will just add a random value for now as GetMagicNumber(comm) requires latest Damaris - // Seed with a real random value, if available - std::random_device r; - - // Choose a random number between 0 and MAX_INT - std::default_random_engine e1(r()); - std::uniform_int_distribution uniform_dist(0, std::numeric_limits::max()); - int rand_int = uniform_dist(e1); - simName_str = "opm-sim-" + std::to_string(rand_int) ; + simName_str = "opm-sim-" + damaris::Environment::GetMagicNumber() ; } else { simName_str = simName ; } @@ -123,6 +193,10 @@ DamarisKeywords(MPI_Comm comm, std::string OutputDir, {"_MAKE_AVAILABLE_IN_PYTHON_",publishToPython_str}, /* must match DamarisKeywords(MPI_Comm comm, std::string OutputDir, bool enableDamarisOutputCollective, diff --git a/opm/simulators/utils/initDamarisXmlFile.cpp b/opm/simulators/utils/initDamarisXmlFile.cpp index 75b6abeaa..4b9d1a80c 100644 --- a/opm/simulators/utils/initDamarisXmlFile.cpp +++ b/opm/simulators/utils/initDamarisXmlFile.cpp @@ -151,17 +151,17 @@ std::string initDamarisXmlFile() - + <_DISABLEPYTHONSTART_pyscript name="PythonScript" file="_PYTHON_SCRIPT_" language="python" frequency="1" scheduler-file="" nthreads="0" keep-workers="no" /_DISABLEPYTHONFIN_> - +<_DISABLEPARAVIEWSTART_paraview update-frequency="1" write-vtk="0" write-vtk-binary="false" > - + - + )V0G0N";