mirror of
https://github.com/OPM/opm-simulators.git
synced 2024-07-07 04:53:03 -05:00
Merge pull request #4451 from akva2/add_serialization_restart_support
Add serialization restart support
This commit is contained in:
Bård Skaflestad
GitHub
commit
0255bcebb1
|
|
@ -449,4 +449,9 @@ endif()
|
|||
|
||||
list (APPEND EXAMPLE_SOURCE_FILES
|
||||
examples/printvfp.cpp
|
||||
)
|
||||
if(HDF5_FOUND)
|
||||
list (APPEND EXAMPLE_SOURCE_FILES
|
||||
examples/opmrst_inspect.cpp
|
||||
)
|
||||
endif()
|
||||
|
|
|
|||
|
|
@ -22,10 +22,10 @@
|
|||
#define ECL_HDF5_SERIALIZER_HH
|
||||
|
||||
#include <opm/common/utility/Serializer.hpp>
|
||||
#include <opm/common/utility/MemPacker.hpp>
|
||||
|
||||
#include <opm/simulators/utils/HDF5File.hpp>
|
||||
#include <opm/simulators/utils/moduleVersion.hpp>
|
||||
#include <opm/simulators/utils/SerializationPackers.hpp>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
|
|
|
|||
77
examples/opmrst_inspect.cpp
Normal file
77
examples/opmrst_inspect.cpp
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
/*
|
||||
Copyright 2020 Equinor.
|
||||
|
||||
This file is part of the Open Porous Media project (OPM).
|
||||
|
||||
OPM is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
OPM is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with OPM. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <config.h>
|
||||
|
||||
#include <ebos/hdf5serializer.hh>
|
||||
|
||||
#include <opm/simulators/timestepping/SimulatorTimer.hpp>
|
||||
|
||||
#include <boost/date_time.hpp>
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
#include <array>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
if (argc < 2) {
|
||||
std::cerr << "Need one parameter, the .OPMRST file to inspect\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
Opm::HDF5Serializer ser(argv[1], Opm::HDF5File::OpenMode::READ);
|
||||
|
||||
std::tuple<std::array<std::string,5>,int> header;
|
||||
try {
|
||||
ser.read(header, "/", "simulator_info");
|
||||
} catch(...) {
|
||||
std::cerr << "Error reading data from file, is it really a .OPMRST file?\n";
|
||||
return 2;
|
||||
}
|
||||
|
||||
const auto& [strings, procs] = header;
|
||||
std::cout << "Info for " << argv[1] <<":\n";
|
||||
std::cout << fmt::format("\tSimulator name: {}\n"
|
||||
"\tSimulator version: {}\n"
|
||||
"\tCompile time stamp: {}\n"
|
||||
"\tCase name: {}\n"
|
||||
"\tNumber of processes: {}\n",
|
||||
strings[0], strings[1], strings[2], strings[3], procs);
|
||||
|
||||
std::cout << fmt::format("\tLast report step: {}\n", ser.lastReportStep());
|
||||
const std::vector<int> reportSteps = ser.reportSteps();
|
||||
for (int step : reportSteps) {
|
||||
Opm::SimulatorTimer timer;
|
||||
try {
|
||||
ser.read(timer, fmt::format("/report_step/{}", step), "simulator_timer");
|
||||
} catch (...) {
|
||||
std::cerr << "*** Failed to read timer info for level " << step << std::endl;
|
||||
}
|
||||
std::cout << "\t\tReport step id " << step << ": Time "
|
||||
<< timer.currentDateTime() << std::endl;
|
||||
}
|
||||
|
||||
std::cout << "====== Parameter values ====\n"
|
||||
<< strings[4];
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -20,6 +20,7 @@ set (opm-simulators_CONFIG_VAR
|
|||
DUNE_ISTL_VERSION_REVISION
|
||||
HAVE_SUITESPARSE_UMFPACK
|
||||
HAVE_DAMARIS
|
||||
HAVE_HDF5
|
||||
)
|
||||
|
||||
# dependencies
|
||||
|
|
|
|||
|
|
@ -38,6 +38,8 @@
|
|||
|
||||
#include <opm/common/ErrorMacros.hpp>
|
||||
|
||||
#include <boost/date_time/gregorian/gregorian.hpp>
|
||||
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
|
@ -46,6 +48,10 @@
|
|||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#if HAVE_HDF5
|
||||
#include <ebos/hdf5serializer.hh>
|
||||
#endif
|
||||
|
||||
namespace Opm::Properties {
|
||||
|
||||
template<class TypeTag, class MyTypeTag>
|
||||
|
|
@ -63,6 +69,24 @@ struct OutputExtraConvergenceInfo
|
|||
using type = UndefinedProperty;
|
||||
};
|
||||
|
||||
template <class TypeTag, class MyTypeTag>
|
||||
struct SaveStep
|
||||
{
|
||||
using type = UndefinedProperty;
|
||||
};
|
||||
|
||||
template <class TypeTag, class MyTypeTag>
|
||||
struct LoadStep
|
||||
{
|
||||
using type = UndefinedProperty;
|
||||
};
|
||||
|
||||
template <class TypeTag, class MyTypeTag>
|
||||
struct SaveFile
|
||||
{
|
||||
using type = UndefinedProperty;
|
||||
};
|
||||
|
||||
template<class TypeTag>
|
||||
struct EnableTerminalOutput<TypeTag, TTag::EclFlowProblem> {
|
||||
static constexpr bool value = true;
|
||||
|
|
@ -82,6 +106,24 @@ struct OutputExtraConvergenceInfo<TypeTag, TTag::EclFlowProblem>
|
|||
static constexpr auto* value = "none";
|
||||
};
|
||||
|
||||
template <class TypeTag>
|
||||
struct SaveStep<TypeTag, TTag::EclFlowProblem>
|
||||
{
|
||||
static constexpr auto* value = "";
|
||||
};
|
||||
|
||||
template <class TypeTag>
|
||||
struct SaveFile<TypeTag, TTag::EclFlowProblem>
|
||||
{
|
||||
static constexpr auto* value = "";
|
||||
};
|
||||
|
||||
template <class TypeTag>
|
||||
struct LoadStep<TypeTag, TTag::EclFlowProblem>
|
||||
{
|
||||
static constexpr int value = -1;
|
||||
};
|
||||
|
||||
} // namespace Opm::Properties
|
||||
|
||||
namespace Opm {
|
||||
|
|
@ -153,6 +195,22 @@ public:
|
|||
OutputExtraConvergenceInfo),
|
||||
R"(OutputExtraConvergenceInfo (--output-extra-convergence-info))");
|
||||
}
|
||||
|
||||
const std::string saveSpec = EWOMS_GET_PARAM(TypeTag, std::string, SaveStep);
|
||||
if (saveSpec == "all") {
|
||||
saveStride_ = 1;
|
||||
} else if (!saveSpec.empty() && saveSpec[0] == ':') {
|
||||
saveStride_ = std::atoi(saveSpec.c_str()+1);
|
||||
} else if (!saveSpec.empty()) {
|
||||
saveStep_ = std::atoi(saveSpec.c_str());
|
||||
}
|
||||
|
||||
loadStep_ = EWOMS_GET_PARAM(TypeTag, int, LoadStep);
|
||||
|
||||
saveFile_ = EWOMS_GET_PARAM(TypeTag, std::string, SaveFile);
|
||||
if (saveFile_.empty()) {
|
||||
saveFile_ = ebosSimulator_.vanguard().caseName() + ".OPMRST";
|
||||
}
|
||||
}
|
||||
|
||||
~SimulatorFullyImplicitBlackoilEbos()
|
||||
|
|
@ -182,6 +240,18 @@ public:
|
|||
"\"iterations\" generates an INFOITER file. "
|
||||
"Combine options with commas, e.g., "
|
||||
"\"steps,iterations\" for multiple outputs.");
|
||||
EWOMS_REGISTER_PARAM(TypeTag, std::string, SaveStep,
|
||||
"Save serialized state to .OPMRST file. "
|
||||
"Either a specific report step, \"all\" to save "
|
||||
"all report steps or \":x\" to save every x'th step.");
|
||||
EWOMS_REGISTER_PARAM(TypeTag, int, LoadStep,
|
||||
"Load serialized state from .OPMRST file. "
|
||||
"Either a specific report step, or 0 to load last "
|
||||
"stored report step.");
|
||||
EWOMS_REGISTER_PARAM(TypeTag, std::string, SaveFile,
|
||||
"FileName for .OPMRST file used for serialized state. "
|
||||
"If empty, CASENAME.OPMRST is used.");
|
||||
EWOMS_HIDE_PARAM(TypeTag, SaveFile);
|
||||
}
|
||||
|
||||
/// Run the simulation.
|
||||
|
|
@ -246,6 +316,10 @@ public:
|
|||
return false;
|
||||
}
|
||||
|
||||
if (loadStep_ > -1) {
|
||||
loadTimerInfo(timer);
|
||||
}
|
||||
|
||||
// Report timestep.
|
||||
if (terminalOutput_) {
|
||||
std::ostringstream ss;
|
||||
|
|
@ -281,7 +355,14 @@ public:
|
|||
+ schedule().seconds(timer.currentStepNum()),
|
||||
timer.currentStepLength());
|
||||
ebosSimulator_.setEpisodeIndex(timer.currentStepNum());
|
||||
if (loadStep_> -1) {
|
||||
wellModel_().prepareDeserialize(loadStep_ - 1);
|
||||
loadSimulatorState();
|
||||
loadStep_ = -1;
|
||||
ebosSimulator_.model().invalidateAndUpdateIntensiveQuantities(/*timeIdx=*/0);
|
||||
}
|
||||
solver->model().beginReportStep();
|
||||
|
||||
bool enableTUNING = EWOMS_GET_PARAM(TypeTag, bool, EnableTuning);
|
||||
|
||||
// If sub stepping is enabled allow the solver to sub cycle
|
||||
|
|
@ -356,6 +437,8 @@ public:
|
|||
OpmLog::debug(msg);
|
||||
}
|
||||
|
||||
handleSave(timer);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -381,6 +464,14 @@ public:
|
|||
const Grid& grid() const
|
||||
{ return ebosSimulator_.vanguard().grid(); }
|
||||
|
||||
template<class Serializer>
|
||||
void serializeOp(Serializer& serializer)
|
||||
{
|
||||
serializer(ebosSimulator_);
|
||||
serializer(report_);
|
||||
serializer(adaptiveTimeStepping_);
|
||||
}
|
||||
|
||||
protected:
|
||||
|
||||
std::unique_ptr<Solver> createSolver(WellModel& wellModel)
|
||||
|
|
@ -471,6 +562,69 @@ protected:
|
|||
this->convergenceOutputThread_->join();
|
||||
}
|
||||
|
||||
//! \brief Serialization of simulator data to .OPMRST files at end of report steps.
|
||||
void handleSave(SimulatorTimer& timer)
|
||||
{
|
||||
if (saveStride_ == -1 && saveStep_ == -1) {
|
||||
return;
|
||||
}
|
||||
|
||||
int nextStep = timer.currentStepNum();
|
||||
|
||||
if ((saveStep_ != -1 && nextStep == saveStep_) ||
|
||||
(saveStride_ != -1 && (nextStep % saveStride_) == 0)) {
|
||||
#if !HAVE_HDF5
|
||||
OpmLog::error("Saving of serialized state requested, but no HDF5 support available.");
|
||||
#else
|
||||
const std::string groupName = "/report_step/" + std::to_string(nextStep);
|
||||
if (nextStep == saveStride_ || nextStep == saveStep_) {
|
||||
std::filesystem::remove(saveFile_);
|
||||
}
|
||||
HDF5Serializer writer(saveFile_, HDF5File::OpenMode::APPEND);
|
||||
if (nextStep == saveStride_ || nextStep == saveStep_) {
|
||||
std::ostringstream str;
|
||||
Parameters::printValues<TypeTag>(str);
|
||||
writer.writeHeader("OPM Flow",
|
||||
moduleVersion(),
|
||||
compileTimestamp(),
|
||||
ebosSimulator_.vanguard().caseName(),
|
||||
str.str(),
|
||||
EclGenericVanguard::comm().size());
|
||||
}
|
||||
writer.write(*this, groupName, "simulator_data");
|
||||
writer.write(timer, groupName, "simulator_timer");
|
||||
OpmLog::info("Serialized state written for report step " + std::to_string(nextStep));
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
//! \brief Load timer info from serialized state.
|
||||
void loadTimerInfo([[maybe_unused]] SimulatorTimer& timer)
|
||||
{
|
||||
#if !HAVE_HDF5
|
||||
OpmLog::error("Loading of serialized state requested, but no HDF5 support available.");
|
||||
loadStep_ = -1;
|
||||
#else
|
||||
HDF5Serializer reader(saveFile_, HDF5File::OpenMode::READ);
|
||||
if (loadStep_ == 0)
|
||||
loadStep_ = reader.lastReportStep();
|
||||
|
||||
OpmLog::info("Loading serialized state for report step " + std::to_string(loadStep_));
|
||||
const std::string groupName = "/report_step/" + std::to_string(loadStep_);
|
||||
reader.read(timer, groupName, "simulator_timer");
|
||||
#endif
|
||||
}
|
||||
|
||||
//! \brief Load simulator state from serialized state.
|
||||
void loadSimulatorState()
|
||||
{
|
||||
#if HAVE_HDF5
|
||||
HDF5Serializer reader(saveFile_, HDF5File::OpenMode::READ);
|
||||
const std::string groupName = "/report_step/" + std::to_string(loadStep_);
|
||||
reader.read(*this, groupName, "simulator_data");
|
||||
#endif
|
||||
}
|
||||
|
||||
// Data.
|
||||
Simulator& ebosSimulator_;
|
||||
std::unique_ptr<WellConnectionAuxiliaryModule<TypeTag>> wellAuxMod_;
|
||||
|
|
@ -491,6 +645,11 @@ protected:
|
|||
std::optional<ConvergenceReportQueue> convergenceOutputQueue_{};
|
||||
std::optional<ConvergenceOutputThread> convergenceOutputObject_{};
|
||||
std::optional<std::thread> convergenceOutputThread_{};
|
||||
|
||||
int saveStride_ = -1; //!< Stride to save serialized state at
|
||||
int saveStep_ = -1; //!< Specific step to save serialized state at
|
||||
int loadStep_ = -1; //!< Step to load serialized state from
|
||||
std::string saveFile_; //!< File to load/save serialized state from/to.
|
||||
};
|
||||
|
||||
} // namespace Opm
|
||||
|
|
|
|||
|
|
@ -228,6 +228,13 @@ namespace Opm {
|
|||
param_.use_multisegment_well_);
|
||||
}
|
||||
|
||||
using BlackoilWellModelGeneric::prepareDeserialize;
|
||||
void prepareDeserialize(const int report_step)
|
||||
{
|
||||
prepareDeserialize(report_step, grid().size(0),
|
||||
param_.use_multisegment_well_);
|
||||
}
|
||||
|
||||
data::Wells wellData() const
|
||||
{
|
||||
auto wsrpt = this->wellState()
|
||||
|
|
|
|||
|
|
@ -218,6 +218,29 @@ initFromRestartFile(const RestartValue& restartValues,
|
|||
initial_step_ = false;
|
||||
}
|
||||
|
||||
void
|
||||
BlackoilWellModelGeneric::
|
||||
prepareDeserialize(int report_step, const size_t numCells, bool handle_ms_well)
|
||||
{
|
||||
// wells_ecl_ should only contain wells on this processor.
|
||||
wells_ecl_ = getLocalWells(report_step);
|
||||
this->local_parallel_well_info_ = createLocalParallelWellInfo(wells_ecl_);
|
||||
|
||||
this->initializeWellProdIndCalculators();
|
||||
initializeWellPerfData();
|
||||
|
||||
if (! this->wells_ecl_.empty()) {
|
||||
handle_ms_well &= anyMSWellOpenLocal();
|
||||
this->wellState().resize(this->wells_ecl_, this->local_parallel_well_info_,
|
||||
this->schedule(), handle_ms_well, numCells,
|
||||
this->well_perf_data_, this->summaryState_);
|
||||
|
||||
}
|
||||
this->wellState().clearWellRates();
|
||||
this->commitWGState();
|
||||
this->updateNupcolWGState();
|
||||
}
|
||||
|
||||
std::vector<Well>
|
||||
BlackoilWellModelGeneric::
|
||||
getLocalWells(const int timeStepIdx) const
|
||||
|
|
|
|||
|
|
@ -138,6 +138,10 @@ public:
|
|||
const size_t numCells,
|
||||
bool handle_ms_well);
|
||||
|
||||
void prepareDeserialize(int report_step,
|
||||
const size_t numCells,
|
||||
bool handle_ms_well);
|
||||
|
||||
/*
|
||||
Will assign the internal member last_valid_well_state_ to the
|
||||
current value of the this->active_well_state_. The state stored
|
||||
|
|
|
|||
|
|
@ -124,6 +124,11 @@ public:
|
|||
return this->well_rates.find(wellName) != this->well_rates.end();
|
||||
}
|
||||
|
||||
void clearWellRates()
|
||||
{
|
||||
this->well_rates.clear();
|
||||
}
|
||||
|
||||
template<class Communication>
|
||||
void gatherVectorsOnRoot(const std::vector< data::Connection >& from_connections,
|
||||
std::vector< data::Connection >& to_connections,
|
||||
|
|
|
|||
|
|
@ -84,3 +84,17 @@ add_test_compare_restarted_simulation(CASENAME spe1
|
|||
REL_TOL ${rel_tol_restart}
|
||||
RESTART_STEP 6
|
||||
TEST_ARGS --sched-restart=false)
|
||||
|
||||
# Serialized restart tests
|
||||
if(HDF5_FOUND)
|
||||
opm_set_test_driver(${PROJECT_SOURCE_DIR}/tests/run-serialization-regressionTest.sh "")
|
||||
add_test_compare_restarted_simulation(CASENAME spe1_serialized
|
||||
DIR spe1
|
||||
FILENAME SPE1CASE1
|
||||
SIMULATOR flow
|
||||
TEST_NAME compareSerializedSim_flow+spe1
|
||||
ABS_TOL 2e-2
|
||||
REL_TOL 1e-5
|
||||
RESTART_STEP 94
|
||||
TEST_ARGS --tolerance-mb=1e-7)
|
||||
endif()
|
||||
|
|
|
|||
63
tests/run-serialization-regressionTest.sh
Executable file
63
tests/run-serialization-regressionTest.sh
Executable file
|
|
@ -0,0 +1,63 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This runs a simulator from start to end, then a restarted
|
||||
# run of the simulator, before comparing the output from the two runs.
|
||||
# This is meant to track regressions in the restart support.
|
||||
|
||||
if test $# -eq 0
|
||||
then
|
||||
echo -e "Usage:\t$0 <options> -- [additional simulator options]"
|
||||
echo -e "\tMandatory options:"
|
||||
echo -e "\t\t -i <path> Path to read deck from"
|
||||
echo -e "\t\t -r <path> Path to store results in"
|
||||
echo -e "\t\t -b <path> Path to simulator binary"
|
||||
echo -e "\t\t -f <filename> Deck file name"
|
||||
echo -e "\t\t -a <tol> Absolute tolerance in comparison"
|
||||
echo -e "\t\t -t <tol> Relative tolerance in comparison"
|
||||
echo -e "\t\t -c <path> Path to comparison tool"
|
||||
echo -e "\t\t -e <filename> Simulator binary to use"
|
||||
echo -e "\t\t -s <step> Step to do restart testing from"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
OPTIND=1
|
||||
while getopts "i:r:b:f:a:t:c:e:d:s:" OPT
|
||||
do
|
||||
case "${OPT}" in
|
||||
i) INPUT_DATA_PATH=${OPTARG} ;;
|
||||
r) RESULT_PATH=${OPTARG} ;;
|
||||
b) BINPATH=${OPTARG} ;;
|
||||
f) FILENAME=${OPTARG} ;;
|
||||
a) ABS_TOL=${OPTARG} ;;
|
||||
t) REL_TOL=${OPTARG} ;;
|
||||
c) COMPARE_ECL_COMMAND=${OPTARG} ;;
|
||||
d) ;;
|
||||
s) RESTART_STEP=${OPTARG} ;;
|
||||
e) EXE_NAME=${OPTARG} ;;
|
||||
esac
|
||||
done
|
||||
shift $(($OPTIND-1))
|
||||
TEST_ARGS="$@"
|
||||
|
||||
BASE_NAME=${FILENAME}
|
||||
|
||||
rm -Rf ${RESULT_PATH}
|
||||
mkdir -p ${RESULT_PATH}
|
||||
cd ${RESULT_PATH}
|
||||
${BINPATH}/${EXE_NAME} ${INPUT_DATA_PATH}/${FILENAME} --output-dir=${RESULT_PATH} ${TEST_ARGS} --save-step=${RESTART_STEP}
|
||||
|
||||
test $? -eq 0 || exit 1
|
||||
|
||||
mkdir -p ${RESULT_PATH}/restart
|
||||
${BINPATH}/${EXE_NAME} ${INPUT_DATA_PATH}/${FILENAME} --output-dir=${RESULT_PATH}/restart ${TEST_ARGS} --load-step=${RESTART_STEP} --save-file=${RESULT_PATH}/${FILENAME}.OPMRST
|
||||
test $? -eq 0 || exit 1
|
||||
|
||||
echo "=== Executing comparison for restart file ==="
|
||||
${COMPARE_ECL_COMMAND} -l -t UNRST ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/restart/${FILENAME} ${ABS_TOL} ${REL_TOL}
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
ecode=1
|
||||
${COMPARE_ECL_COMMAND} -a -l -t UNRST ${RESULT_PATH}/${FILENAME} ${RESULT_PATH}/restart/${FILENAME} ${ABS_TOL} ${REL_TOL}
|
||||
fi
|
||||
|
||||
exit $ecode
|
||||
Loading…
Reference in New Issue
Block a user