diff --git a/compareECLFiles.cmake b/compareECLFiles.cmake index 27d5f4e93b..7a75c0587c 100644 --- a/compareECLFiles.cmake +++ b/compareECLFiles.cmake @@ -408,6 +408,13 @@ add_test_runSimulator(CASENAME spe1case1_carfin DIR lgr TEST_ARGS --parsing-strictness=low --enable-ecl-output=false --enable-vtk-output=true) +add_test_runSimulator(CASENAME spe1case1_carfin_parallel + FILENAME SPE1CASE1_CARFIN + SIMULATOR flow + DIR lgr + PROCS 4 + TEST_ARGS --parsing-strictness=low --enable-ecl-output=false --enable-vtk-output=true) + # Tests that are run based on simulator results, but not necessarily direct comparison to reference results add_test_runSimulator(CASENAME tuning_xxxmbe FILENAME 01_TUNING_XXXMBE diff --git a/opm/simulators/flow/CollectDataOnIORank_impl.hpp b/opm/simulators/flow/CollectDataOnIORank_impl.hpp index 7b8c450c87..91e0a25d86 100644 --- a/opm/simulators/flow/CollectDataOnIORank_impl.hpp +++ b/opm/simulators/flow/CollectDataOnIORank_impl.hpp @@ -851,8 +851,8 @@ CollectDataOnIORank(const Grid& grid, const EquilGrid* equilGrid, : toIORankComm_(grid.comm()) , globalInterRegFlows_(InterRegFlowMap::createMapFromNames(toVector(fipRegionsInterregFlow))) { - // index maps only have to be build when reordering is needed - if (!needsReordering && !isParallel()) + // Build index maps only when reordering is needed; skip in parallel runs for CpGrid with LGRs + if ((!needsReordering && !isParallel()) || (isParallel() && (grid.maxLevel()>0))) return; const CollectiveCommunication& comm = grid.comm(); diff --git a/opm/simulators/flow/CpGridVanguard.hpp b/opm/simulators/flow/CpGridVanguard.hpp index e4faf1755a..8cf1980de5 100644 --- a/opm/simulators/flow/CpGridVanguard.hpp +++ b/opm/simulators/flow/CpGridVanguard.hpp @@ -263,19 +263,26 @@ public: */ void addLgrs() { - // Check if input file contains Lgrs. - // - // If there are lgrs, create the grid with them, and update the leaf grid view. - if (const auto& lgrs = this->eclState().getLgrs(); (lgrs.size() > 0) && (this->grid_->comm().size() == 1)) { + // Check if input file contains Lgrs. Add them, if any. + // In a parallel run, this adds the LGRs on the distributed simulation grid. + if (const auto& lgrs = this->eclState().getLgrs(); lgrs.size() > 0) { OpmLog::info("\nAdding LGRs to the grid and updating its leaf grid view"); this->addLgrsUpdateLeafView(lgrs, lgrs.size(), *this->grid_); this->updateGridView_(); this->updateCellDepths_(); this->updateCellThickness_(); - } - else if (this->grid_->comm().size() > 1) { - OpmLog::warning("Adding LGRs in parallel run is not supported yet.\n"); + + if (this->grid_->comm().size()>1) { + // Add LGRs and update the leaf grid view in the global (undistributed) simulation grid. + // Purpose: To enable synchronization of cell ids in 'serial mode', + // we rely on the "parent-to-children" cell id mapping. + OpmLog::info("\nAdding LGRs to the global view and updating its leaf grid view"); + this->grid_->switchToGlobalView(); + this->addLgrsUpdateLeafView(lgrs, lgrs.size(), *this->grid_); + this->grid_->switchToDistributedView(); + this->grid_->syncDistributedGlobalCellIds(); + } } } diff --git a/opm/simulators/flow/GenericCpGridVanguard.cpp b/opm/simulators/flow/GenericCpGridVanguard.cpp index b9c9a7f657..c2adeae1b2 100644 --- a/opm/simulators/flow/GenericCpGridVanguard.cpp +++ b/opm/simulators/flow/GenericCpGridVanguard.cpp @@ -33,6 +33,7 @@ #include #include +#include #include #include @@ -306,7 +307,13 @@ distributeFieldProps_(EclipseState& eclState1) { // Reset Cartesian index mapper for automatic creation of field // properties - parallelEclState->resetCartesianMapper(this->cartesianIndexMapper_.get()); + // Note: the previous cartesianIndexMapper_ has been replaces by levelCartesianIndexMapper_ + // to support also the case of a distributed level zero grid in a CpGrid with LGRs. + // This change allows access to the Cartesian indices of the distributed level zero grid, + // where the field properties are given - for now. + // In case of supporting LGR field properties, this need to be adapted, to access instead + // each local/level Cartesian index set. + parallelEclState->resetCartesianMapper(this->levelCartesianIndexMapper_.get()); parallelEclState->switchToDistributedProps(); } else { @@ -500,6 +507,7 @@ void GenericCpGridVanguard::doCreateGrids_(Eclips } cartesianIndexMapper_ = std::make_unique(*grid_); + levelCartesianIndexMapper_ = std::make_unique(*grid_); #if HAVE_MPI if (this->grid_->comm().size() > 1) { @@ -640,7 +648,7 @@ template const LevelCartesianIndexMapper GenericCpGridVanguard::levelCartesianIndexMapper() const { - return LevelCartesianIndexMapper(*grid_); + return *levelCartesianIndexMapper_; } template diff --git a/opm/simulators/flow/GenericCpGridVanguard.hpp b/opm/simulators/flow/GenericCpGridVanguard.hpp index b9042bb9ab..b29b8cab55 100644 --- a/opm/simulators/flow/GenericCpGridVanguard.hpp +++ b/opm/simulators/flow/GenericCpGridVanguard.hpp @@ -233,6 +233,7 @@ protected: std::unique_ptr equilGrid_; std::unique_ptr cartesianIndexMapper_; std::unique_ptr equilCartesianIndexMapper_; + std::unique_ptr levelCartesianIndexMapper_; int mpiRank; std::vector cell_part_{}; diff --git a/opm/simulators/flow/OutputBlackoilModule.hpp b/opm/simulators/flow/OutputBlackoilModule.hpp index 4d1932f56f..0709cd78db 100644 --- a/opm/simulators/flow/OutputBlackoilModule.hpp +++ b/opm/simulators/flow/OutputBlackoilModule.hpp @@ -673,6 +673,11 @@ private: void createLocalRegion_(std::vector& region) { + // For CpGrid with LGRs, where level zero grid has been distributed, + // resize region is needed, since in this case the total amount of + // element - per process - in level zero grid and leaf grid do not + // coincide, in general. + region.resize(simulator_.gridView().size(0)); std::size_t elemIdx = 0; for (const auto& elem : elements(simulator_.gridView())) { if (elem.partitionType() != Dune::InteriorEntity) { diff --git a/opm/simulators/utils/ParallelEclipseState.hpp b/opm/simulators/utils/ParallelEclipseState.hpp index 95c05a1f9a..ad0f0ac273 100644 --- a/opm/simulators/utils/ParallelEclipseState.hpp +++ b/opm/simulators/utils/ParallelEclipseState.hpp @@ -102,9 +102,14 @@ public: template void resetCartesianMapper(const T* mapper) { - m_activeSize = std::bind(&T::compressedSize, mapper); - m_local2Global = std::bind(&T::cartesianIndex, mapper, - std::placeholders::_1); + // Note: mapper would usually be a CartesianIndexMapper. However, to support also + // the case of a distributed level zero grid in a CpGrid with LGRs, mapper will be + // Opm::LevelCartesianIndexMapper. This change allows access to the Cartesian indices + // of the distributed level zero grid, where the field properties are given - for now. + // In case of supporting LGR field properties, this need to be adapted, to access instead + // each local/level Cartesian index set. + m_activeSize = [mapper]() { return mapper->compressedSize(/*level = */ 0); }; + m_local2Global = [mapper](int localIdx) { return mapper->cartesianIndex(localIdx, /* level = */ 0); }; } bool tran_active(const std::string& keyword) const override;