OilfieldToolsNet/opm-simulators
4
0
Fork 0
mirror of https://github.com/OPM/opm-simulators.git synced 2025-02-25 18:55:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

Changes needed to add Damaris functionality. The current state is that we output the pressure field and we use both HDF5 and parallel HDF5.

Browse Source 
Damaris initialization is added after InitMpi but before starting the simulation. Damaris will invoke a separate core for writing in
parallel and leave the rest of cores for the simulator. The main changes are in main where start_damaris and then in eclwriterm where
we use damaris to output the PRESSURE. To test Damaris one can use --enable-damaris-output=true and to use parallel HDF5 one can use
--enable-async-damaris-output=true (false is the default choice)
This commit is contained in:
Joshua Bowden
2021-10-22 11:09:12 +02:00
committed by Elyes Ahmed
parent 7e1c63c92d
commit fa7af3540c
11 changed files with 471 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
opm/simulators/utils/DamarisKeywords.cpp Normal file
View File

@@ -0,0 +1,63 @@
/*
Copyright 2021 Equinor.
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include <opm/simulators/utils/DamarisKeywords.hpp>
#include <string>
#include <map>
/*
Below is the Damaris Keywords supported by Damaris to be filled
in the built-in XML file.
The entries in the map below will be filled by the corresponding
Damaris Keywords. Yet, only output directory and FileMode are to
be chosen by the user
*/
namespace Opm::DamarisOutput
{
std::map<std::string,std::string> DamarisKeywords(std::string OutputDir, bool enableAsyncDamarisOutput) {
if (enableAsyncDamarisOutput) {
std::map<std::string,std::string> damaris_keywords = {
{"_SHMEM_BUFFER_BYTES_REGEX_","536870912"},
{"_DC_REGEX_","1"},
{"_DN_REGEX_","0"},
{"_File_Mode", "Collective"},
{"_MORE_VARIABLES_REGEX_",""},
{"_PATH_REGEX_", OutputDir},
{"_MYSTORE_OR_EMPTY_REGEX_","MyStore"},
};
return damaris_keywords;
}
else {
std::map<std::string,std::string> damaris_keywords = {
{"_SHMEM_BUFFER_BYTES_REGEX_","536870912"},
{"_DC_REGEX_","1"},
{"_DN_REGEX_","0"},
{"_File_Mode", "FilePerCore"},
{"_MORE_VARIABLES_REGEX_",""},
{"_File_Mode", OutputDir},
{"_MYSTORE_OR_EMPTY_REGEX_","MyStore"},
};
return damaris_keywords;
}
}
}

42
opm/simulators/utils/DamarisKeywords.hpp Normal file
View File

@@ -0,0 +1,42 @@
/*
Copyright 2021 Equinor.
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef OPM_DAMARISKEYWORDS_HEADER_INCLUDED
#define OPM_DAMARISKEYWORDS_HEADER_INCLUDED
#include <string>
#include <map>
/*
Below is the std::map with the keywords that are supported by Damaris.
Most entries in the map below are not critical ('static') and will not
be changed. We only allow changing FileMode together with output directory
*/
namespace Opm::DamarisOutput
{
std::map<std::string,std::string> DamarisKeywords(std::string OutputDir, bool enableAsyncDamarisOutput);
} // namespace Opm::DamarisOutput
#endif

173
opm/simulators/utils/DamarisOutputModule.cpp Normal file
View File

@@ -0,0 +1,173 @@
/*
Copyright 2022 SINTEF Digital, Mathematics and Cybernetics.
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include <config.h>
#include <damaris/model/ModifyModel.hpp>
#include <opm/simulators/utils/DamarisOutputModule.hpp>
#include <opm/simulators/utils/DamarisKeywords.hpp>
#include <opm/simulators/utils/ParallelCommunication.hpp>
/*
Below is the XML file for Damaris that is supported by Damaris.
The entries in the map below will be filled by corresponding Damaris
Keywords.
*/
namespace Opm::DamarisOutput
{
std::string initDamarisXmlFile()
{
std::string init_damaris = R"V0G0N(<?xml version="1.0"?>
<simulation name="opm-flow" language="c"
xmlns="http://damaris.gforge.inria.fr/damaris/model">
<architecture>
<domains count="1"/>
<dedicated cores="_DC_REGEX_" nodes="_DN_REGEX_"/>
<buffer name="buffer" size="_SHMEM_BUFFER_BYTES_REGEX_" />
<placement />
<queue name="queue" size="300" />
</architecture>
<data>
<parameter name="n_elements_total" type="int" value="1" />
<parameter name="n_elements_local" type="int" value="1" />
<parameter name="n" type="int" value="1" />
<layout name="zonal_layout_usmesh" type="double" dimensions="n_elements_local" global="n_elements_total" comment="For the field data e.g. Pressure" />
<variable name="PRESSURE" layout="zonal_layout_usmesh" type="scalar" visualizable="false" unit="Pa" centering="zonal" store="_MYSTORE_OR_EMPTY_REGEX_" />
_MORE_VARIABLES_REGEX_
</data>
<storage>
<store name="MyStore" type="HDF5">
<option key="FileMode">"_File_Mode"</option>
<option key="XDMFMode">NoIteration</option>
<option key="FilesPath">./</option>
</store>
</storage>
<actions>
</actions>
<log FileName="_PATH_REGEX_/damaris_log/exa_dbg" RotationSize="5" LogFormat="[%TimeStamp%]: %Message%" Flush="True" LogLevel="debug" />
</simulation>)V0G0N";
return init_damaris;
}
void initializeDamaris(MPI_Comm comm, int mpiRank, std::string OutputDir, bool enableAsyncDamarisOutput)
{
// Prepare the XML file
std::string damaris_config_xml = initDamarisXmlFile();
damaris::model::ModifyModel myMod = damaris::model::ModifyModel(damaris_config_xml);
// The map will make it precise the output directory and FileMode (either FilePerCore or Collective storage)
// The map file find all occurences of the string in position 1 and repalce it/them with string in position 2
std::map<std::string,std::string> find_replace_map = DamarisKeywords(OutputDir, enableAsyncDamarisOutput);
myMod.RepalceWithRegEx(find_replace_map);
std::string damaris_xml_filename_str = OutputDir +"/damaris_config.xml" ;
if (mpiRank == 0) {
myMod.SaveXMLStringToFile(damaris_xml_filename_str);
}
int damaris_err;
/* Get the name of the Damaris input file from an environment variable if available */
const char *cs_damaris_xml_file = getenv("FLOW_DAMARIS_XML_FILE");
if (cs_damaris_xml_file != NULL)
{
std::cout << "INFO: initializing Damaris from environment variable FLOW_DAMARIS_XML_FILE: " << cs_damaris_xml_file << std::endl;
damaris_err = damaris_initialize(cs_damaris_xml_file, MPI_COMM_WORLD);
if (damaris_err != DAMARIS_OK ) {
std::cerr << "ERROR: damaris_initialize() error via FLOW_DAMARIS_XML_FILE=" << cs_damaris_xml_file << std::endl;
}
}
else
{
std::cout << "INFO: initializing Damaris using internally built file:" << damaris_xml_filename_str << std::endl;
damaris_err = damaris_initialize(damaris_xml_filename_str.c_str() , comm);
if (damaris_err != DAMARIS_OK ) {
std::cerr << "ERROR: damaris_initialize() error via built file:" << std::endl << myMod.GetConfigString();
}
}
}
void setupDamarisWritingPars(Parallel::Communication comm, const int n_elements_local_grid)
{
int damaris_err = DAMARIS_OK;
const int nranks = comm.size();
const int rank = comm.rank();
std::vector<unsigned long long> elements_rank_sizes(nranks); // one for each rank -- to be gathered from each client rank
std::vector<unsigned long long> elements_rank_offsets(nranks); // one for each rank, first one 0 -- to be computed - Probably could use MPI_Scan()!
// n_elements_local_grid should be the full model size
const unsigned long long n_elements_local = n_elements_local_grid ;
std::cout << "INFO (" << rank << "): n_elements_local_grid = " << n_elements_local_grid << std::endl ;
// This gets the n_elements_local from all ranks and copies them to a std::vector of all the values on all ranks (elements_rank_sizes[]).
comm.allgather(&n_elements_local, 1, elements_rank_sizes.data());
elements_rank_offsets[0] = 0ULL ;
// This scan makes the offsets to the start of each ranks grid section if each local grid data was concatenated (in rank order)
for (int t1 = 1 ; t1 < nranks; t1++) {
elements_rank_offsets[t1] = elements_rank_offsets[t1-1] + elements_rank_sizes[t1-1];
}
// find the global/total size
unsigned long long n_elements_global_max = elements_rank_offsets[nranks-1] ;
n_elements_global_max += elements_rank_sizes[nranks-1] ; // add the last ranks size to the already accumulated offset values
if (rank == 0 ) {
std::cout << "INFO (" << rank << "): n_elements_global_max = " << n_elements_global_max << std::endl ;
}
// Set the paramater so that the Damaris servers can allocate the correct amount of memory for the variabe
// Damaris parameters only support int data types. This will limit models to be under size of 2^32-1 elements
// ToDo: Do we need to check that local ranks are 0 based ?
int temp_int = static_cast<int>(elements_rank_sizes[rank]) ;
damaris_err = damaris_parameter_set("n_elements_local",&temp_int, sizeof(int));
if (damaris_err != DAMARIS_OK ) {
std::cerr << "ERROR: Damaris library produced an error result for damaris_parameter_set(n_elements_local,&temp_int, sizeof(int));" << std::endl ;
}
// Damaris parameters only support int data types. This will limit models to be under size of 2^32-1 elements
// ToDo: Do we need to check that n_elements_global_max will fit in a C int type (INT_MAX)
temp_int = static_cast<int>(n_elements_global_max) ;
damaris_err = damaris_parameter_set("n_elements_total",&temp_int, sizeof(int));
if (damaris_err != DAMARIS_OK ) {
std::cerr << "ERROR: Damaris library produced an error result for damaris_parameter_set(n_elements_local,&temp_int, sizeof(int));" << std::endl ;
}
// Use damaris_set_position to set the offset in the global size of the array.
// This is used so that output functionality (e.g. HDF5Store) knows global offsets of the data of the ranks
int64_t temp_int64_t[1] ;
temp_int64_t[0] = static_cast<int64_t>(elements_rank_offsets[rank]) ;
damaris_err = damaris_set_position("PRESSURE",temp_int64_t) ;
if (damaris_err != DAMARIS_OK ) {
std::cerr << "ERROR: Damaris library produced an error result for damaris_set_position(\"PRESSURE\",temp_int64_t);" << std::endl ;
}
}
}

41
opm/simulators/utils/DamarisOutputModule.hpp Normal file
View File

@@ -0,0 +1,41 @@
/*
Copyright 2022 SINTEF Digital, Mathematics and Cybernetics.
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include <string>
#include <Damaris.h>
#include <opm/simulators/utils/ParallelCommunication.hpp>
/*
Below is the XML file for Damaris that is supported by Damaris.
The entries in the map below will be filled by corresponding Damaris
Keywords.
*/
namespace Opm::DamarisOutput
{
// Initialize an XML file
std::string initDamarisXmlFile();
// Initialize Damaris by filling in th XML file and stroring it in the chosed directory
void initializeDamaris(MPI_Comm comm, int mpiRank, std::string OutputDir, bool enableAsyncDamarisOutput);
// Setup Damaris Parameters for writing e.g., grid size and communicator to output "PRESSURE" field
void setupDamarisWritingPars(Parallel::Communication comm, const int n_elements_local_grid);
} // namespace Opm::DamarisOutput