diff --git a/opm/simulators/flow/ReservoirCouplingMaster.cpp b/opm/simulators/flow/ReservoirCouplingMaster.cpp index a6a11de1a..3113dd039 100644 --- a/opm/simulators/flow/ReservoirCouplingMaster.cpp +++ b/opm/simulators/flow/ReservoirCouplingMaster.cpp @@ -50,6 +50,37 @@ ReservoirCouplingMaster::ReservoirCouplingMaster( // Public methods // ------------------ +void ReservoirCouplingMaster::receiveSimulationStartDateFromSlaves() { + this->slave_start_dates_.resize(this->num_slaves_); + if (this->comm_.rank() == 0) { + // Ensure that std::time_t is of type long since we are sending it over MPI with MPI_LONG + static_assert(std::is_same::value, "std::time_t is not of type long"); + for (unsigned int i = 0; i < this->master_slave_comm_.size(); i++) { + std::time_t start_date; + int result = MPI_Recv( + &start_date, + /*count=*/1, + /*datatype=*/MPI_LONG, + /*source_rank=*/0, + /*tag=*/static_cast(MessageTag::SimulationStartDate), + *this->master_slave_comm_[i].get(), + MPI_STATUS_IGNORE + ); + if (result != MPI_SUCCESS) { + OPM_THROW(std::runtime_error, "Failed to receive simulation start date from slave process"); + } + this->slave_start_dates_[i] = start_date; + OpmLog::info( + fmt::format( + "Received simulation start date from slave process with name: {}. " + "Start date: {}", this->slave_names_[i], start_date + ) + ); + } + } + this->comm_.broadcast(this->slave_start_dates_.data(), this->num_slaves_, /*emitter_rank=*/0); +} + // NOTE: This functions is executed for all ranks, but only rank 0 will spawn // the slave processes void ReservoirCouplingMaster::spawnSlaveProcesses(int argc, char **argv) { @@ -101,6 +132,7 @@ void ReservoirCouplingMaster::spawnSlaveProcesses(int argc, char **argv) { ); this->master_slave_comm_.push_back(std::move(master_slave_comm)); this->slave_names_.push_back(slave_name); + this->num_slaves_++; } } diff --git a/opm/simulators/flow/ReservoirCouplingMaster.hpp b/opm/simulators/flow/ReservoirCouplingMaster.hpp index e9091b7bd..6d1ed57b1 100644 --- a/opm/simulators/flow/ReservoirCouplingMaster.hpp +++ b/opm/simulators/flow/ReservoirCouplingMaster.hpp @@ -34,12 +34,13 @@ namespace Opm { class ReservoirCouplingMaster { public: + using MPI_Comm_Ptr = ReservoirCoupling::MPI_Comm_Ptr; + using MessageTag = ReservoirCoupling::MessageTag; ReservoirCouplingMaster(const Parallel::Communication &comm, const Schedule &schedule); void spawnSlaveProcesses(int argc, char **argv); - - using MPI_Comm_Ptr = ReservoirCoupling::MPI_Comm_Ptr; + void receiveSimulationStartDateFromSlaves(); private: std::vector getSlaveArgv( @@ -52,9 +53,10 @@ private: const Parallel::Communication &comm_; const Schedule& schedule_; - // MPI communicators for the slave processes - std::vector master_slave_comm_; + std::size_t num_slaves_ = 0; // Initially zero, will be updated in spawnSlaveProcesses() + std::vector master_slave_comm_; // MPI communicators for the slave processes std::vector slave_names_; + std::vector slave_start_dates_; }; } // namespace Opm diff --git a/opm/simulators/flow/ReservoirCouplingSlave.cpp b/opm/simulators/flow/ReservoirCouplingSlave.cpp index 690214158..f369b0bf1 100644 --- a/opm/simulators/flow/ReservoirCouplingSlave.cpp +++ b/opm/simulators/flow/ReservoirCouplingSlave.cpp @@ -38,17 +38,30 @@ ReservoirCouplingSlave::ReservoirCouplingSlave( ) : comm_{comm}, schedule_{schedule} -{ } -void ReservoirCouplingSlave::sendSimulationStartDateToMasterProcess() { - // TODO: Implement this function next +{ + this->slave_master_comm_ = MPI_Comm_Ptr(new MPI_Comm(MPI_COMM_NULL)); + MPI_Comm_get_parent(this->slave_master_comm_.get()); + if (*(this->slave_master_comm_) == MPI_COMM_NULL) { + OPM_THROW(std::runtime_error, "Slave process is not spawned by a master process"); + } +} - //this->slave_master_comm_ = MPI_Comm_Ptr(new MPI_Comm(MPI_COMM_NULL)); - //MPI_Comm_get_parent(this->slave_master_comm_.get()); - //if (*(this->slave_master_comm_) == MPI_COMM_NULL) { - // OPM_THROW(std::runtime_error, "Slave process is not spawned by a master process"); - //} - //MPI_Send(&start_date, 1, MPI_INT, 0, 0, parentcomm); - OpmLog::info("Sent simulation start date to master process"); +void ReservoirCouplingSlave::sendSimulationStartDateToMasterProcess() { + + if (this->comm_.rank() == 0) { + // Ensure that std::time_t is of type long since we are sending it over MPI with MPI_LONG + static_assert(std::is_same::value, "std::time_t is not of type long"); + std::time_t start_date = this->schedule_.getStartTime(); + MPI_Send( + &start_date, + /*count=*/1, + /*datatype=*/MPI_LONG, + /*dest_rank=*/0, + /*tag=*/static_cast(MessageTag::SimulationStartDate), + *this->slave_master_comm_ + ); + OpmLog::info("Sent simulation start date to master process from rank 0"); + } } } // namespace Opm diff --git a/opm/simulators/flow/ReservoirCouplingSlave.hpp b/opm/simulators/flow/ReservoirCouplingSlave.hpp index 12f1d9590..e34b8cfeb 100644 --- a/opm/simulators/flow/ReservoirCouplingSlave.hpp +++ b/opm/simulators/flow/ReservoirCouplingSlave.hpp @@ -20,7 +20,7 @@ #ifndef OPM_RESERVOIR_COUPLING_SLAVE_HPP #define OPM_RESERVOIR_COUPLING_SLAVE_HPP -#include +#include #include #include #include @@ -33,7 +33,8 @@ namespace Opm { class ReservoirCouplingSlave { public: - using MPI_Comm_Ptr = ReservoirCouplingMaster::MPI_Comm_Ptr; + using MPI_Comm_Ptr = ReservoirCoupling::MPI_Comm_Ptr; + using MessageTag = ReservoirCoupling::MessageTag; ReservoirCouplingSlave(const Parallel::Communication &comm, const Schedule &schedule); void sendSimulationStartDateToMasterProcess(); diff --git a/opm/simulators/flow/SimulatorFullyImplicitBlackoil.hpp b/opm/simulators/flow/SimulatorFullyImplicitBlackoil.hpp index 1f62bb95b..1a8dbfb40 100644 --- a/opm/simulators/flow/SimulatorFullyImplicitBlackoil.hpp +++ b/opm/simulators/flow/SimulatorFullyImplicitBlackoil.hpp @@ -218,16 +218,15 @@ public: // For now, we require that SLAVES and GRUPMAST are defined at the first // schedule step, so it is enough to check the first step. See the // keyword handlers in opm-common for more information. - if (!this->schedule()[0].rescoup.empty()) { - auto master_mode = this->schedule()[0].rescoup().masterMode(); - if (master_mode) { - this->reservoirCouplingMaster_ = - std::make_unique( - FlowGenericVanguard::comm(), - this->schedule() - ); - this->reservoirCouplingMaster_->spawnSlaveProcesses(argc, argv); - } + auto master_mode = this->schedule()[0].rescoup().masterMode(); + if (master_mode) { + this->reservoirCouplingMaster_ = + std::make_unique( + FlowGenericVanguard::comm(), + this->schedule() + ); + this->reservoirCouplingMaster_->spawnSlaveProcesses(argc, argv); + this->reservoirCouplingMaster_->receiveSimulationStartDateFromSlaves(); } } #endif