diff --git a/flow/flow.cpp b/flow/flow.cpp index 38d751cd3..956cd4bfc 100644 --- a/flow/flow.cpp +++ b/flow/flow.cpp @@ -352,18 +352,14 @@ int main(int argc, char** argv) Opm::FlowMainEbos::printPRTHeader(outputCout); -#ifdef HAVE_MPI - Opm::ParallelEclipseState* parState; -#endif if (mpiRank == 0) { deck.reset( new Opm::Deck( parser.parseFile(deckFilename , parseContext, errorGuard))); Opm::MissingFeatures::checkKeywords(*deck, parseContext, errorGuard); if ( outputCout ) Opm::checkDeck(*deck, parser, parseContext, errorGuard); -#ifdef HAVE_MPI - parState = new Opm::ParallelEclipseState(*deck); - eclipseState.reset(parState); +#if HAVE_MPI + eclipseState.reset(new Opm::ParallelEclipseState(*deck)); #else eclipseState.reset(new Opm::EclipseState(*deck)); #endif @@ -385,21 +381,17 @@ int main(int argc, char** argv) setupMessageLimiter(schedule->getMessageLimits(), "STDOUT_LOGGER"); summaryConfig.reset( new Opm::SummaryConfig(*deck, *schedule, eclipseState->getTableManager(), parseContext, errorGuard)); -#ifdef HAVE_MPI - Opm::Mpi::packAndSend(*schedule, Dune::MPIHelper::getCollectiveCommunication()); -#endif } -#ifdef HAVE_MPI +#if HAVE_MPI else { summaryConfig.reset(new Opm::SummaryConfig); schedule.reset(new Opm::Schedule); - parState = new Opm::ParallelEclipseState; - Opm::Mpi::receiveAndUnpack(*schedule, mpiHelper.getCollectiveCommunication()); - eclipseState.reset(parState); + eclipseState.reset(new Opm::ParallelEclipseState); } Opm::EclMpiSerializer ser(mpiHelper.getCollectiveCommunication()); ser.broadcast(*summaryConfig); - ser.broadcast(*parState); + ser.broadcast(*eclipseState); + ser.broadcast(*schedule); #endif Opm::checkConsistentArrayDimensions(*eclipseState, *schedule, parseContext, errorGuard); diff --git a/flow/flow_tag.hpp b/flow/flow_tag.hpp index 845b17c12..2797ca6c9 100644 --- a/flow/flow_tag.hpp +++ b/flow/flow_tag.hpp @@ -51,6 +51,11 @@ #include #endif +#if HAVE_MPI +#include +#include +#endif + BEGIN_PROPERTIES @@ -65,11 +70,13 @@ END_PROPERTIES namespace Opm { template - void flowEbosSetDeck(Deck &deck, EclipseState& eclState) + void flowEbosSetDeck(Deck *deck, EclipseState& eclState, Schedule& schedule, SummaryConfig& summaryConfig) { typedef typename GET_PROP_TYPE(TypeTag, Vanguard) Vanguard; - Vanguard::setExternalDeck(&deck); + Vanguard::setExternalDeck(deck); Vanguard::setExternalEclState(&eclState); + Vanguard::setExternalSchedule(&schedule); + Vanguard::setExternalSummaryConfig(&summaryConfig); } // ----------------- Main program ----------------- @@ -371,15 +378,32 @@ int mainFlow(int argc, char** argv) Opm::FlowMainEbos::printPRTHeader(outputCout); - deck.reset( new Opm::Deck( parser.parseFile(deckFilename , parseContext, errorGuard))); - Opm::MissingFeatures::checkKeywords(*deck, parseContext, errorGuard); - if ( outputCout ) - Opm::checkDeck(*deck, parser, parseContext, errorGuard); + if (mpiRank == 0) { + deck.reset( new Opm::Deck( parser.parseFile(deckFilename , parseContext, errorGuard))); + Opm::MissingFeatures::checkKeywords(*deck, parseContext, errorGuard); + if ( outputCout ) + Opm::checkDeck(*deck, parser, parseContext, errorGuard); - eclipseState.reset( new Opm::EclipseState(*deck )); - schedule.reset(new Opm::Schedule(*deck, *eclipseState, parseContext, errorGuard)); - summaryConfig.reset( new Opm::SummaryConfig(*deck, *schedule, eclipseState->getTableManager(), parseContext, errorGuard)); - setupMessageLimiter(schedule->getMessageLimits(), "STDOUT_LOGGER"); +#if HAVE_MPI + eclipseState.reset(new Opm::ParallelEclipseState(*deck)); +#else + eclipseState.reset(new Opm::EclipseState(*deck)); +#endif + schedule.reset(new Opm::Schedule(*deck, *eclipseState, parseContext, errorGuard)); + setupMessageLimiter(schedule->getMessageLimits(), "STDOUT_LOGGER"); + summaryConfig.reset( new Opm::SummaryConfig(*deck, *schedule, eclipseState->getTableManager(), parseContext, errorGuard)); + } +#if HAVE_MPI + else { + summaryConfig.reset(new Opm::SummaryConfig); + schedule.reset(new Opm::Schedule); + eclipseState.reset(new Opm::ParallelEclipseState); + } + Opm::EclMpiSerializer ser(mpiHelper.getCollectiveCommunication()); + ser.broadcast(*summaryConfig); + ser.broadcast(*eclipseState); + ser.broadcast(*schedule); +#endif Opm::checkConsistentArrayDimensions(*eclipseState, *schedule, parseContext, errorGuard); @@ -391,7 +415,7 @@ int mainFlow(int argc, char** argv) } } bool outputFiles = (outputMode != FileOutputMode::OUTPUT_NONE); - Opm::flowEbosSetDeck(*deck, *eclipseState); + Opm::flowEbosSetDeck(deck.get(), *eclipseState, *schedule, *summaryConfig); return Opm::flowEbosMain(argc, argv, outputCout, outputFiles); } catch (const std::invalid_argument& e) diff --git a/opm/simulators/utils/ParallelEclipseState.cpp b/opm/simulators/utils/ParallelEclipseState.cpp index 899d25867..d11a57235 100644 --- a/opm/simulators/utils/ParallelEclipseState.cpp +++ b/opm/simulators/utils/ParallelEclipseState.cpp @@ -19,8 +19,6 @@ #include #include "ParallelEclipseState.hpp" -#include "ParallelRestart.hpp" -#include namespace Opm { diff --git a/opm/simulators/utils/ParallelRestart.cpp b/opm/simulators/utils/ParallelRestart.cpp index 19f3605f5..823a87e8c 100644 --- a/opm/simulators/utils/ParallelRestart.cpp +++ b/opm/simulators/utils/ParallelRestart.cpp @@ -50,7 +50,6 @@ #include #include #include -#include #include #include #include @@ -176,49 +175,6 @@ void reconstructDynState(const std::vector& unique, result = Opm::DynamicState(ptrData, idxVec.back()); } -template class Map, class Type, class Key> -void reconstructDynMap(const std::vector& unique, - const std::vector>>& asMap, - Map>& result) -{ - for (const auto& it : asMap) { - reconstructDynState(unique, it.second, result[it.first]); - } -} - -template class Map, class Type, class Key> -std::size_t packSizeDynMap(const Map>& data, - Dune::MPIHelper::MPICommunicator comm) -{ - auto split = splitDynMap(data); - return Opm::Mpi::packSize(split.first, comm) + - Opm::Mpi::packSize(split.second, comm); -} - -template class Map, class Type, class Key> -void packDynMap(const Map>& data, - std::vector& buffer, - int& position, - Dune::MPIHelper::MPICommunicator comm) -{ - auto split = splitDynMap(data); - Opm::Mpi::pack(split.first, buffer, position, comm); - Opm::Mpi::pack(split.second, buffer, position, comm); -} - -template class Map, class Type, class Key> -void unpackDynMap(Map>& data, - std::vector& buffer, - int& position, - Dune::MPIHelper::MPICommunicator comm) -{ - std::vector unique; - std::vector>> indices; - Opm::Mpi::unpack(unique, buffer, position, comm); - Opm::Mpi::unpack(indices, buffer, position, comm); - reconstructDynMap(unique, indices, data); -} - } namespace Opm @@ -1534,37 +1490,6 @@ std::size_t packSize(const Action::Actions& data, return packSize(data.getActions(), comm); } -template using Map2 = std::map; - -std::size_t packSize(const Schedule& data, - Dune::MPIHelper::MPICommunicator comm) -{ - return packSize(data.getTimeMap(), comm) + - packSizeDynMap(data.getStaticWells(), comm) + - packSizeDynMap(data.getGroups(), comm) + - packSize(data.getOilVapProps(), comm) + - packSize(data.getEvents(), comm) + - packSize(data.getModifierDeck(), comm) + - packSize(data.getTuning(), comm) + - packSize(data.getMessageLimits(), comm) + - packSize(data.getRunspec(), comm) + - packSizeDynMap(data.getVFPProdTables(), comm) + - packSizeDynMap(data.getVFPInjTables(), comm) + - packSize(data.getWellTestConfig(), comm) + - packSize(data.getWListManager(), comm) + - packSize(data.getUDQConfig(), comm) + - packSize(data.getUDQActive(), comm) + - packSize(data.getGuideRateConfig(), comm) + - packSize(data.getGConSale(), comm) + - packSize(data.getGConSump(), comm) + - packSize(data.getGlobalWhistCtlMode(), comm) + - packSize(data.getActions(), comm) + - packSize(data.rftConfig(), comm) + - packSize(data.getNupCol(), comm) + - packSize(data.restart(), comm) + - packSize(data.getWellGroupEvents(), comm); -} - std::size_t packSize(const BrineDensityTable& data, Dune::MPIHelper::MPICommunicator comm) { @@ -3160,36 +3085,6 @@ void pack(const Action::Actions& data, pack(data.getActions(), buffer, position, comm); } -void pack(const Schedule& data, - std::vector& buffer, int& position, - Dune::MPIHelper::MPICommunicator comm) -{ - pack(data.getTimeMap(), buffer, position, comm); - packDynMap(data.getStaticWells(), buffer, position, comm); - packDynMap(data.getGroups(), buffer, position, comm); - pack(data.getOilVapProps(), buffer, position, comm); - pack(data.getEvents(), buffer, position, comm); - pack(data.getModifierDeck(), buffer, position, comm); - pack(data.getTuning(), buffer, position, comm); - pack(data.getMessageLimits(), buffer, position, comm); - pack(data.getRunspec(), buffer, position, comm); - packDynMap(data.getVFPProdTables(), buffer, position, comm); - packDynMap(data.getVFPInjTables(), buffer, position, comm); - pack(data.getWellTestConfig(), buffer, position, comm); - pack(data.getWListManager(), buffer, position, comm); - pack(data.getUDQConfig(), buffer, position, comm); - pack(data.getUDQActive(), buffer, position, comm); - pack(data.getGuideRateConfig(), buffer, position, comm); - pack(data.getGConSale(), buffer, position, comm); - pack(data.getGConSump(), buffer, position, comm); - pack(data.getGlobalWhistCtlMode(), buffer, position, comm); - pack(data.getActions(), buffer, position, comm); - pack(data.rftConfig(), buffer, position, comm); - pack(data.getNupCol(), buffer, position, comm); - pack(data.restart(), buffer, position, comm); - pack(data.getWellGroupEvents(), buffer, position, comm); -} - void pack(const BrineDensityTable& data, std::vector& buffer, int& position, Dune::MPIHelper::MPICommunicator comm) @@ -5411,67 +5306,6 @@ void unpack(Action::Actions& data, std::vector& buffer, int& position, data = Action::Actions(actions); } -void unpack(Schedule& data, std::vector& buffer, int& position, - Dune::MPIHelper::MPICommunicator comm) -{ - TimeMap timeMap; - Schedule::WellMap staticWells; - Schedule::GroupMap groups; - DynamicState oilVapProps; - Events events; - DynamicVector modifierDeck; - DynamicState tuning; - MessageLimits messageLimits; - Runspec runspec; - Schedule::VFPProdMap vfpProdTables; - Schedule::VFPInjMap vfpInjTables; - DynamicState> wellTestConfig; - DynamicState> wListManager; - DynamicState> udqConfig; - DynamicState> udqActive; - DynamicState> guideRateConfig; - DynamicState> gconSale; - DynamicState> gconSump; - DynamicState globalWhistCtlMode; - DynamicState> actions; - RFTConfig rftConfig; - DynamicState nupCol; - RestartConfig restartConfig; - std::map wellGroupEvents; - - unpack(timeMap, buffer, position, comm); - unpackDynMap(staticWells, buffer, position, comm); - unpackDynMap(groups, buffer, position, comm); - unpack(oilVapProps, buffer, position, comm); - unpack(events, buffer, position, comm); - unpack(modifierDeck, buffer, position, comm); - unpack(tuning, buffer, position, comm); - unpack(messageLimits, buffer, position, comm); - unpack(runspec, buffer, position, comm); - unpackDynMap(vfpProdTables, buffer, position, comm); - unpackDynMap(vfpInjTables, buffer, position, comm); - unpack(wellTestConfig, buffer, position, comm); - unpack(wListManager, buffer, position, comm); - unpack(udqConfig, buffer, position, comm); - unpack(udqActive, buffer, position, comm); - unpack(guideRateConfig, buffer, position, comm); - unpack(gconSale, buffer, position, comm); - unpack(gconSump, buffer, position, comm); - unpack(globalWhistCtlMode, buffer, position, comm); - unpack(actions, buffer, position, comm); - - unpack(rftConfig, buffer, position, comm); - unpack(nupCol, buffer, position, comm); - unpack(restartConfig, buffer, position, comm); - unpack(wellGroupEvents, buffer, position, comm); - data = Schedule(timeMap, staticWells, groups, oilVapProps, events, - modifierDeck, tuning, messageLimits, runspec, - vfpProdTables, vfpInjTables, wellTestConfig, - wListManager, udqConfig, udqActive, guideRateConfig, - gconSale, gconSump, globalWhistCtlMode, actions, - rftConfig, nupCol, restartConfig, wellGroupEvents); -} - void unpack(BrineDensityTable& data, std::vector& buffer, int& position, Dune::MPIHelper::MPICommunicator comm) { @@ -5845,6 +5679,13 @@ INSTANTIATE_PACK_VECTOR(bool) INSTANTIATE_PACK_VECTOR(char) INSTANTIATE_PACK_VECTOR(int) INSTANTIATE_PACK_VECTOR(std::array) +INSTANTIATE_PACK_VECTOR(std::pair) +INSTANTIATE_PACK_VECTOR(std::shared_ptr) +INSTANTIATE_PACK_VECTOR(std::shared_ptr) +INSTANTIATE_PACK_VECTOR(std::shared_ptr) +INSTANTIATE_PACK_VECTOR(std::shared_ptr) +INSTANTIATE_PACK_VECTOR(std::pair>) +INSTANTIATE_PACK_VECTOR(std::pair>) #undef INSTANTIATE_PACK_VECTOR @@ -5873,6 +5714,9 @@ template void unpack(std::shared_ptr<__VA_ARGS__>& data, \ Dune::MPIHelper::MPICommunicator comm); INSTANTIATE_PACK_SHARED_PTR(SpiralICD) +INSTANTIATE_PACK_SHARED_PTR(VFPInjTable) +INSTANTIATE_PACK_SHARED_PTR(Well) +INSTANTIATE_PACK_SHARED_PTR(WellTestConfig) #undef INSTANTIATE_PACK_SHARED_PTR #define INSTANTIATE_PACK(...) \ @@ -5892,6 +5736,28 @@ INSTANTIATE_PACK(int) INSTANTIATE_PACK(std::array) INSTANTIATE_PACK(std::array) INSTANTIATE_PACK(unsigned char) +INSTANTIATE_PACK(std::map,std::pair>) +INSTANTIATE_PACK(std::map) +INSTANTIATE_PACK(std::map>) +INSTANTIATE_PACK(std::map) +INSTANTIATE_PACK(std::map>) +INSTANTIATE_PACK(std::map,int>>) +INSTANTIATE_PACK(std::unordered_map) +INSTANTIATE_PACK(std::pair) +INSTANTIATE_PACK(DynamicState) +INSTANTIATE_PACK(DynamicState) +INSTANTIATE_PACK(DynamicState>) +INSTANTIATE_PACK(DynamicState>) +INSTANTIATE_PACK(DynamicState>) +INSTANTIATE_PACK(DynamicState>) +INSTANTIATE_PACK(DynamicState>) +INSTANTIATE_PACK(DynamicState>) +INSTANTIATE_PACK(DynamicState) +INSTANTIATE_PACK(DynamicState) +INSTANTIATE_PACK(DynamicState>) +INSTANTIATE_PACK(DynamicState>) +INSTANTIATE_PACK(DynamicVector) + #undef INSTANTIATE_PACK } // end namespace Mpi diff --git a/opm/simulators/utils/ParallelRestart.hpp b/opm/simulators/utils/ParallelRestart.hpp index 5d5c90b3e..3e4db769e 100644 --- a/opm/simulators/utils/ParallelRestart.hpp +++ b/opm/simulators/utils/ParallelRestart.hpp @@ -128,7 +128,6 @@ class RocktabTable; class Rock2dTable; class Rock2dtrTable; class Runspec; -class Schedule; class Segment; class ShrateRecord; class ShrateTable; @@ -598,7 +597,6 @@ ADD_PACK_PROTOTYPES(Rock2dTable) ADD_PACK_PROTOTYPES(Rock2dtrTable) ADD_PACK_PROTOTYPES(RocktabTable) ADD_PACK_PROTOTYPES(Runspec) -ADD_PACK_PROTOTYPES(Schedule) ADD_PACK_PROTOTYPES(Segment) ADD_PACK_PROTOTYPES(ShrateRecord) ADD_PACK_PROTOTYPES(ShrateTable) @@ -661,31 +659,6 @@ ADD_PACK_PROTOTYPES(WellTracerProperties) ADD_PACK_PROTOTYPES(WList) ADD_PACK_PROTOTYPES(WListManager) -template -const T& packAndSend(const T& in, const C& comm) -{ - if (comm.size() == 1) - return in; - - std::size_t size = packSize(in, comm); - std::vector buffer(size); - int pos = 0; - Mpi::pack(in, buffer, pos, comm); - comm.broadcast(&pos, 1, 0); - comm.broadcast(buffer.data(), pos, 0); - return in; -} - -template -void receiveAndUnpack(T& result, const C& comm) -{ - int size; - comm.broadcast(&size, 1, 0); - std::vector buffer(size); - comm.broadcast(buffer.data(), size, 0); - int pos = 0; - unpack(result, buffer, pos, comm); -} } // end namespace Mpi RestartValue loadParallelRestart(const EclipseIO* eclIO, SummaryState& summaryState, diff --git a/tests/test_ParallelRestart.cpp b/tests/test_ParallelRestart.cpp index 7da702519..3b8ac58fe 100644 --- a/tests/test_ParallelRestart.cpp +++ b/tests/test_ParallelRestart.cpp @@ -2161,7 +2161,7 @@ BOOST_AUTO_TEST_CASE(Schedule) getRestartConfig(), {{"test", events}}); - auto val2 = PackUnpack(val1); + auto val2 = PackUnpack2(val1); DO_CHECKS(Schedule) #endif }