OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

Update for support of Damaris mesh based in situ processing

Browse Source 
Moved damaris command line parameter accessors to Main.cpp and fixed support for both Python and Paraview Python scripts, as they both cannot be present in the same simulation (seems to be an initialization conflict or double initialization)
This commit is contained in:
josh bowden 2023-09-23 22:24:11 +02:00
parent 8095eb0c48
commit bb38c57b1b
5 changed files with 144 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
opm/simulators/flow/Main.cpp
View File

@ -233,12 +233,48 @@ void Main::setupVanguard()
}
#if HAVE_DAMARIS
void Main::setupDamaris(const std::string& outputDir , std::map<std::string, std::string>& find_replace_map)
void Main::setupDamaris(const std::string& outputDir )
{
typedef Properties::TTag::FlowEarlyBird PreTypeTag;
if (!outputDir.empty()) {
ensureOutputDirExists(outputDir);
}
bool enableDamarisOutputCollective = true ;
bool saveToDamarisHDF5 = true ;
std::string damarisPythonFilename = "" ;
std::string damarisPythonParaviewFilename = "" ;
std::string damarisSimName = "" ; // empty defaults to opm-sim-<magic_number>
std::string damarisLogLevel = "info" ;
int nDamarisCores = 1 ;
int nDamarisNodes = 0 ;
long shmemSizeBytes = 536870912 ;
enableDamarisOutputCollective = EWOMS_GET_PARAM(PreTypeTag, bool, EnableDamarisOutputCollective) ;
saveToDamarisHDF5 = EWOMS_GET_PARAM(PreTypeTag, bool, DamarisSaveToHdf);
damarisPythonFilename = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisPythonScript);
damarisPythonParaviewFilename = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisPythonParaviewScript);
damarisSimName = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisSimName);
nDamarisCores = EWOMS_GET_PARAM(PreTypeTag, int, DamarisDedicatedCores);
nDamarisNodes = EWOMS_GET_PARAM(PreTypeTag, int, DamarisDedicatedNodes);
shmemSizeBytes = EWOMS_GET_PARAM(PreTypeTag, long, DamarisSharedMemeorySizeBytes);
damarisLogLevel = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisLogLevel);
std::map<std::string, std::string> find_replace_map ;
find_replace_map = Opm::DamarisOutput::DamarisKeywords(EclGenericVanguard::comm(),
outputDir,
enableDamarisOutputCollective,
saveToDamarisHDF5,
nDamarisCores,
nDamarisNodes,
shmemSizeBytes,
damarisPythonFilename,
damarisSimName,
damarisLogLevel,
damarisPythonParaviewFilename
);
// By default EnableDamarisOutputCollective is true so all simulation results will
// be written into one single file for each iteration using Parallel HDF5.
// It set to false, FilePerCore mode is used in Damaris, then simulation results in each

42
opm/simulators/flow/Main.hpp
View File

@ -342,34 +342,9 @@ private:
OpmLog::warning(msg);
enableDamarisOutput_ = false ;
}
if (enableDamarisOutput_) {
enableDamarisOutputCollective_ = EWOMS_GET_PARAM(PreTypeTag, bool, EnableDamarisOutputCollective) ;
saveToDamarisHDF5_ = EWOMS_GET_PARAM(PreTypeTag, bool, DamarisSaveToHdf);
damarisPythonFilename_ = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisPythonScript);
damarisPythonParaviewFilename_ = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisPythonParaviewScript);
damarisSimName_ = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisSimName);
nDamarisCores_ = EWOMS_GET_PARAM(PreTypeTag, int, DamarisDedicatedCores);
nDamarisNodes_ = EWOMS_GET_PARAM(PreTypeTag, int, DamarisDedicatedNodes);
shmemSizeBytes_ = EWOMS_GET_PARAM(PreTypeTag, long, DamarisSharedMemeorySizeBytes);
damarisLogLevel_ = EWOMS_GET_PARAM(PreTypeTag, std::string, DamarisLogLevel);
std::map<std::string, std::string> find_replace_map ;
find_replace_map = Opm::DamarisOutput::DamarisKeywords(EclGenericVanguard::comm(),
outputDir,
enableDamarisOutputCollective_,
saveToDamarisHDF5_,
nDamarisCores_,
nDamarisNodes_,
shmemSizeBytes_,
damarisPythonFilename_,
damarisSimName_,
damarisLogLevel_,
damarisPythonParaviewFilename_
);
this->setupDamaris(outputDir, find_replace_map);
this->setupDamaris(outputDir);
}
#endif // HAVE_DAMARIS
@ -737,8 +712,7 @@ private:
}
#if HAVE_DAMARIS
void setupDamaris(const std::string& outputDir,
std::map<std::string, std::string>& find_replace_map);
void setupDamaris(const std::string& outputDir);
#endif
int argc_{0};
@ -763,16 +737,6 @@ private:
bool isSimulationRank_ = true;
#if HAVE_DAMARIS
bool enableDamarisOutput_ = false;
bool enableDamarisOutputCollective_ = true ;
bool saveToDamarisHDF5_ = true ;
std::string damarisPythonFilename_ = "" ;
std::string damarisPythonParaviewFilename_ = "" ;
std::string damarisSimName_ = "" ; // empty defaults to opm-sim-<magic_number>
std::string damarisLogLevel_ = "info" ;
int nDamarisCores_ = 1 ;
int nDamarisNodes_ = 0 ;
long shmemSizeBytes_ = 536870912 ;
#endif
};

104
opm/simulators/utils/DamarisKeywords.cpp
View File

@ -18,13 +18,16 @@
*/
#include <opm/simulators/utils/DamarisKeywords.hpp>
#include <damaris/env/Environment.hpp>
#include <string>
#include <map>
#include <random>
#include <fstream>
#include <Damaris.h>
#include <mpi.h>
/*
Below is the Damaris Keywords supported by Damaris to be filled
Below are the Damaris Keywords supported by Damaris to be filled
in the built-in XML file.
The entries in the map below will be filled by the corresponding
@ -34,6 +37,41 @@
namespace Opm::DamarisOutput
{
bool FileExists(const std::string filename_in, const MPI_Comm comm, const int rank) {
// From c++17 : std::filesystem::exists(filename_in);
int retint = 0 ;
std::ifstream filestr ;
bool file_exists = false ;
if ((filename_in.length() == 0) || (filename_in == "#") ) {
return file_exists ;
}
if (rank == 0) {
filestr.open(filename_in);
file_exists = true ;
if(filestr.fail()) {
retint = 0 ;
} else {
retint = 1 ;
filestr.close() ;
}
MPI_Bcast(&retint,1,MPI_INT,0,comm);
} else {
MPI_Bcast(&retint,1,MPI_INT,0,comm); // recieve the value from rank 0
}
if (retint == 1) {
file_exists = true ;
}
return (file_exists) ;
}
std::map<std::string, std::string>
DamarisKeywords(MPI_Comm comm, std::string OutputDir,
bool enableDamarisOutputCollective,
@ -46,11 +84,53 @@ DamarisKeywords(MPI_Comm comm, std::string OutputDir,
std::string logLevel,
std::string paraviewPythonFilename )
{
int rank ;
MPI_Comm_rank(comm, &rank) ;
std::string saveToHDF5_str("MyStore") ;
if (! saveToHDF5 ) saveToHDF5_str = "" ;
if (! saveToHDF5 ) saveToHDF5_str = "#" ;
std::string disablePythonXMLstart("!--") ;
std::string disablePythonXMLfin("--") ;
std::string disableParaviewXMLstart("!--") ;
std::string disableParaviewXMLfin("--") ;
// Test if input Python file exists and set the name of the script for <variable ... script="" > )XML elements
std::string publishToPython_str("") ;
if (pythonFilename != "") publishToPython_str="PythonScript" ; // the name of the PyScript XML element
if (pythonFilename != ""){
if (FileExists(pythonFilename, comm, rank)) {
publishToPython_str="PythonScript" ; // the name of the PyScript XML element
disablePythonXMLstart = std::string("") ;
disablePythonXMLfin = std::string("") ;
} else {
pythonFilename = "" ; // set to empty if it does not exist
std::string disablePythonXMLstart("!--") ;
std::string disablePythonXMLfin("--") ;
}
}
// Test if input Paraview Python file exists
if (paraviewPythonFilename != ""){
if (FileExists(paraviewPythonFilename, comm, rank)) {
disableParaviewXMLstart = std::string("") ;
disableParaviewXMLfin = std::string("") ;
} else {
paraviewPythonFilename = "" ; // set to empty if it does not exist
disableParaviewXMLstart = std::string("!--") ;
disableParaviewXMLfin = std::string("--") ;
}
}
// Flag error if both scripts are enabled
// It would be good to know if Damaris has either one compiled into the library - this is currently not possible
if ((pythonFilename.size() > 0) && (paraviewPythonFilename.size() > 0) )
{
std::cerr << "ERROR: Both the Python (--damaris-python-script command line argument) and Paraview Python " <<
"(--damaris-python-paraview-script command line argument) scripts are valid, however only one type "
"of analysis is supported in a single simulation. Please choose one. Exiting." << std::endl ;
std::exit(-1) ;
}
std::string damarisOutputCollective_str("") ;
if (enableDamarisOutputCollective) {
@ -64,17 +144,7 @@ DamarisKeywords(MPI_Comm comm, std::string OutputDir,
// Having a different simulation name is important if multiple simulations
// are running on the same node, as it is used to name the simulations shmem area
// and when one sim finishes it removes its shmem file.
// simName_str = "opm-sim-" + damaris::Environment::GetMagicNumber(comm) ;
// We will just add a random value for now as GetMagicNumber(comm) requires latest Damaris
// Seed with a real random value, if available
std::random_device r;
// Choose a random number between 0 and MAX_INT
std::default_random_engine e1(r());
std::uniform_int_distribution<int> uniform_dist(0, std::numeric_limits<int>::max());
int rand_int = uniform_dist(e1);
simName_str = "opm-sim-" + std::to_string(rand_int) ;
simName_str = "opm-sim-" + damaris::Environment::GetMagicNumber() ;
} else {
simName_str = simName ;
}
@ -123,6 +193,10 @@ DamarisKeywords(MPI_Comm comm, std::string OutputDir,
{"_MAKE_AVAILABLE_IN_PYTHON_",publishToPython_str}, /* must match <pyscript name="PythonScript" */
{"_SIM_NAME_",simName_str},
{"_LOG_LEVEL_",logLevel_str},
{"_DISABLEPYTHONSTART_",disablePythonXMLstart},
{"_DISABLEPYTHONFIN_",disablePythonXMLfin},
{"_DISABLEPARAVIEWSTART_",disableParaviewXMLstart},
{"_DISABLEPARAVIEWFIN_",disableParaviewXMLfin},
};
return damaris_keywords;

11
opm/simulators/utils/DamarisKeywords.hpp
View File

@ -33,7 +33,18 @@
namespace Opm::DamarisOutput
{
/**
* Returns true if the file exists.
* Tests to see if filename string is empty
* or the "#" character and if so returns false.
*/
bool FileExists(const std::string filename_in, const MPI_Comm comm, const int rank) ;
/**
* Returns true if the file exists. Tests to see if filename string is empty or the "#" character and if so returns false.
*/
std::map<std::string, std::string>
DamarisKeywords(MPI_Comm comm, std::string OutputDir,
bool enableDamarisOutputCollective,

8
opm/simulators/utils/initDamarisXmlFile.cpp
View File

@ -151,17 +151,17 @@ std::string initDamarisXmlFile()
</storage>
<scripts>
<pyscript name="PythonScript" file="_PYTHON_SCRIPT_" language="python" frequency="1" scheduler-file="" nthreads="0" keep-workers="no" />
<_DISABLEPYTHONSTART_pyscript name="PythonScript" file="_PYTHON_SCRIPT_" language="python" frequency="1" scheduler-file="" nthreads="0" keep-workers="no" /_DISABLEPYTHONFIN_>
</scripts>
<paraview update-frequency="1" write-vtk="0" write-vtk-binary="false" >
<_DISABLEPARAVIEWSTART_paraview update-frequency="1" write-vtk="0" write-vtk-binary="false" >
<script>_PARAVIEW_PYTHON_SCRIPT_</script>
</paraview>
</paraview _DISABLEPARAVIEWFIN_>
<actions>
</actions>
<log FileName="_PATH_REGEX_/damaris_log/_SIM_NAME_" RotationSize="5" LogFormat="[%TimeStamp%]: %Message%" Flush="True" LogLevel="_LOG_LEVEL_" />
<log FileName="_PATH_REGEX_/damaris_log/_SIM_NAME_" RotationSize="5" LogFormat="[%TimeStamp%]: %Message%" Flush="false" LogLevel="_LOG_LEVEL_" />
</simulation>)V0G0N";