diff --git a/opm/simulators/flow/Main.cpp b/opm/simulators/flow/Main.cpp index f29ed5d48..04965a442 100644 --- a/opm/simulators/flow/Main.cpp +++ b/opm/simulators/flow/Main.cpp @@ -233,12 +233,48 @@ void Main::setupVanguard() } #if HAVE_DAMARIS -void Main::setupDamaris(const std::string& outputDir , std::map& find_replace_map) +void Main::setupDamaris(const std::string& outputDir ) { + typedef Properties::TTag::FlowEarlyBird PreTypeTag; if (!outputDir.empty()) { ensureOutputDirExists(outputDir); } + bool enableDamarisOutputCollective = true ; + bool saveToDamarisHDF5 = true ; + std::string damarisPythonFilename = "" ; + std::string damarisPythonParaviewFilename = "" ; + + std::string damarisSimName = "" ; // empty defaults to opm-sim- + std::string damarisLogLevel = "info" ; + int nDamarisCores = 1 ; + int nDamarisNodes = 0 ; + long shmemSizeBytes = 536870912 ; + + enableDamarisOutputCollective = EWOMS_GET_PARAM(PreTypeTag, bool, EnableDamarisOutputCollective) ; + saveToDamarisHDF5 = EWOMS_GET_PARAM(PreTypeTag, bool, DamarisSaveToHdf); + damarisPythonFilename = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisPythonScript); + damarisPythonParaviewFilename = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisPythonParaviewScript); + damarisSimName = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisSimName); + nDamarisCores = EWOMS_GET_PARAM(PreTypeTag, int, DamarisDedicatedCores); + nDamarisNodes = EWOMS_GET_PARAM(PreTypeTag, int, DamarisDedicatedNodes); + shmemSizeBytes = EWOMS_GET_PARAM(PreTypeTag, long, DamarisSharedMemeorySizeBytes); + damarisLogLevel = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisLogLevel); + + std::map find_replace_map ; + find_replace_map = Opm::DamarisOutput::DamarisKeywords(EclGenericVanguard::comm(), + outputDir, + enableDamarisOutputCollective, + saveToDamarisHDF5, + nDamarisCores, + nDamarisNodes, + shmemSizeBytes, + damarisPythonFilename, + damarisSimName, + damarisLogLevel, + damarisPythonParaviewFilename + ); + // By default EnableDamarisOutputCollective is true so all simulation results will // be written into one single file for each iteration using Parallel HDF5. // It set to false, FilePerCore mode is used in Damaris, then simulation results in each diff --git a/opm/simulators/flow/Main.hpp b/opm/simulators/flow/Main.hpp index 333117790..561d75910 100644 --- a/opm/simulators/flow/Main.hpp +++ b/opm/simulators/flow/Main.hpp @@ -342,34 +342,9 @@ private: OpmLog::warning(msg); enableDamarisOutput_ = false ; } - + if (enableDamarisOutput_) { - enableDamarisOutputCollective_ = EWOMS_GET_PARAM(PreTypeTag, bool, EnableDamarisOutputCollective) ; - saveToDamarisHDF5_ = EWOMS_GET_PARAM(PreTypeTag, bool, DamarisSaveToHdf); - damarisPythonFilename_ = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisPythonScript); - damarisPythonParaviewFilename_ = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisPythonParaviewScript); - damarisSimName_ = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisSimName); - - nDamarisCores_ = EWOMS_GET_PARAM(PreTypeTag, int, DamarisDedicatedCores); - nDamarisNodes_ = EWOMS_GET_PARAM(PreTypeTag, int, DamarisDedicatedNodes); - shmemSizeBytes_ = EWOMS_GET_PARAM(PreTypeTag, long, DamarisSharedMemeorySizeBytes); - - damarisLogLevel_ = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisLogLevel); - - std::map find_replace_map ; - find_replace_map = Opm::DamarisOutput::DamarisKeywords(EclGenericVanguard::comm(), - outputDir, - enableDamarisOutputCollective_, - saveToDamarisHDF5_, - nDamarisCores_, - nDamarisNodes_, - shmemSizeBytes_, - damarisPythonFilename_, - damarisSimName_, - damarisLogLevel_, - damarisPythonParaviewFilename_ - ); - this->setupDamaris(outputDir, find_replace_map); + this->setupDamaris(outputDir); } #endif // HAVE_DAMARIS @@ -737,8 +712,7 @@ private: } #if HAVE_DAMARIS - void setupDamaris(const std::string& outputDir, - std::map& find_replace_map); + void setupDamaris(const std::string& outputDir); #endif int argc_{0}; @@ -763,16 +737,6 @@ private: bool isSimulationRank_ = true; #if HAVE_DAMARIS bool enableDamarisOutput_ = false; - bool enableDamarisOutputCollective_ = true ; - bool saveToDamarisHDF5_ = true ; - std::string damarisPythonFilename_ = "" ; - std::string damarisPythonParaviewFilename_ = "" ; - - std::string damarisSimName_ = "" ; // empty defaults to opm-sim- - std::string damarisLogLevel_ = "info" ; - int nDamarisCores_ = 1 ; - int nDamarisNodes_ = 0 ; - long shmemSizeBytes_ = 536870912 ; #endif }; diff --git a/opm/simulators/utils/DamarisKeywords.cpp b/opm/simulators/utils/DamarisKeywords.cpp index 1564fb540..88a1d2a06 100644 --- a/opm/simulators/utils/DamarisKeywords.cpp +++ b/opm/simulators/utils/DamarisKeywords.cpp @@ -18,13 +18,16 @@ */ #include +#include #include #include -#include +#include +#include + #include /* - Below is the Damaris Keywords supported by Damaris to be filled + Below are the Damaris Keywords supported by Damaris to be filled in the built-in XML file. The entries in the map below will be filled by the corresponding @@ -34,6 +37,41 @@ namespace Opm::DamarisOutput { + +bool FileExists(const std::string filename_in, const MPI_Comm comm, const int rank) { + // From c++17 : std::filesystem::exists(filename_in); + + int retint = 0 ; + std::ifstream filestr ; + bool file_exists = false ; + + if ((filename_in.length() == 0) || (filename_in == "#") ) { + return file_exists ; + } + + if (rank == 0) { + + filestr.open(filename_in); + file_exists = true ; + if(filestr.fail()) { + retint = 0 ; + } else { + retint = 1 ; + filestr.close() ; + } + MPI_Bcast(&retint,1,MPI_INT,0,comm); + } else { + MPI_Bcast(&retint,1,MPI_INT,0,comm); // recieve the value from rank 0 + } + + if (retint == 1) { + file_exists = true ; + } + + return (file_exists) ; +} + + std::map DamarisKeywords(MPI_Comm comm, std::string OutputDir, bool enableDamarisOutputCollective, @@ -46,11 +84,53 @@ DamarisKeywords(MPI_Comm comm, std::string OutputDir, std::string logLevel, std::string paraviewPythonFilename ) { + int rank ; + MPI_Comm_rank(comm, &rank) ; std::string saveToHDF5_str("MyStore") ; - if (! saveToHDF5 ) saveToHDF5_str = "" ; + if (! saveToHDF5 ) saveToHDF5_str = "#" ; + + std::string disablePythonXMLstart("!--") ; + std::string disablePythonXMLfin("--") ; + std::string disableParaviewXMLstart("!--") ; + std::string disableParaviewXMLfin("--") ; + + + // Test if input Python file exists and set the name of the script for )XML elements std::string publishToPython_str("") ; - if (pythonFilename != "") publishToPython_str="PythonScript" ; // the name of the PyScript XML element + if (pythonFilename != ""){ + if (FileExists(pythonFilename, comm, rank)) { + publishToPython_str="PythonScript" ; // the name of the PyScript XML element + disablePythonXMLstart = std::string("") ; + disablePythonXMLfin = std::string("") ; + } else { + pythonFilename = "" ; // set to empty if it does not exist + std::string disablePythonXMLstart("!--") ; + std::string disablePythonXMLfin("--") ; + } + } + + // Test if input Paraview Python file exists + if (paraviewPythonFilename != ""){ + if (FileExists(paraviewPythonFilename, comm, rank)) { + disableParaviewXMLstart = std::string("") ; + disableParaviewXMLfin = std::string("") ; + } else { + paraviewPythonFilename = "" ; // set to empty if it does not exist + disableParaviewXMLstart = std::string("!--") ; + disableParaviewXMLfin = std::string("--") ; + } + } + + // Flag error if both scripts are enabled + // It would be good to know if Damaris has either one compiled into the library - this is currently not possible + if ((pythonFilename.size() > 0) && (paraviewPythonFilename.size() > 0) ) + { + std::cerr << "ERROR: Both the Python (--damaris-python-script command line argument) and Paraview Python " << + "(--damaris-python-paraview-script command line argument) scripts are valid, however only one type " + "of analysis is supported in a single simulation. Please choose one. Exiting." << std::endl ; + std::exit(-1) ; + } std::string damarisOutputCollective_str("") ; if (enableDamarisOutputCollective) { @@ -64,17 +144,7 @@ DamarisKeywords(MPI_Comm comm, std::string OutputDir, // Having a different simulation name is important if multiple simulations // are running on the same node, as it is used to name the simulations shmem area // and when one sim finishes it removes its shmem file. - // simName_str = "opm-sim-" + damaris::Environment::GetMagicNumber(comm) ; - - // We will just add a random value for now as GetMagicNumber(comm) requires latest Damaris - // Seed with a real random value, if available - std::random_device r; - - // Choose a random number between 0 and MAX_INT - std::default_random_engine e1(r()); - std::uniform_int_distribution uniform_dist(0, std::numeric_limits::max()); - int rand_int = uniform_dist(e1); - simName_str = "opm-sim-" + std::to_string(rand_int) ; + simName_str = "opm-sim-" + damaris::Environment::GetMagicNumber() ; } else { simName_str = simName ; } @@ -123,6 +193,10 @@ DamarisKeywords(MPI_Comm comm, std::string OutputDir, {"_MAKE_AVAILABLE_IN_PYTHON_",publishToPython_str}, /* must match DamarisKeywords(MPI_Comm comm, std::string OutputDir, bool enableDamarisOutputCollective, diff --git a/opm/simulators/utils/initDamarisXmlFile.cpp b/opm/simulators/utils/initDamarisXmlFile.cpp index 75b6abeaa..4b9d1a80c 100644 --- a/opm/simulators/utils/initDamarisXmlFile.cpp +++ b/opm/simulators/utils/initDamarisXmlFile.cpp @@ -151,17 +151,17 @@ std::string initDamarisXmlFile() - + <_DISABLEPYTHONSTART_pyscript name="PythonScript" file="_PYTHON_SCRIPT_" language="python" frequency="1" scheduler-file="" nthreads="0" keep-workers="no" /_DISABLEPYTHONFIN_> - +<_DISABLEPARAVIEWSTART_paraview update-frequency="1" write-vtk="0" write-vtk-binary="false" > - + - + )V0G0N";