diff --git a/CMakeLists_files.cmake b/CMakeLists_files.cmake index c1be9a96d..ada38e4aa 100644 --- a/CMakeLists_files.cmake +++ b/CMakeLists_files.cmake @@ -23,7 +23,6 @@ # originally generated with the command: # find opm -name '*.c*' -printf '\t%p\n' | sort list (APPEND MAIN_SOURCE_FILES - ebos/collecttoiorank.cc ebos/eclgenericcpgridvanguard.cc ebos/eclgenericproblem.cc ebos/eclgenericthresholdpressure.cc @@ -41,6 +40,7 @@ list (APPEND MAIN_SOURCE_FILES opm/simulators/timestepping/SimulatorReport.cpp opm/simulators/flow/ActionHandler.cpp opm/simulators/flow/Banners.cpp + opm/simulators/flow/CollectDataOnIORank.cpp opm/simulators/flow/ConvergenceOutputConfiguration.cpp opm/simulators/flow/ExtraConvergenceOutputThread.cpp opm/simulators/flow/FlowMain.cpp @@ -50,12 +50,12 @@ list (APPEND MAIN_SOURCE_FILES opm/simulators/flow/LogOutputHelper.cpp opm/simulators/flow/Main.cpp opm/simulators/flow/NonlinearSolver.cpp + opm/simulators/flow/partitionCells.cpp opm/simulators/flow/RSTConv.cpp opm/simulators/flow/RegionPhasePVAverage.cpp opm/simulators/flow/SimulatorReportBanners.cpp opm/simulators/flow/SimulatorSerializer.cpp opm/simulators/flow/ValidationFunctions.cpp - opm/simulators/flow/partitionCells.cpp opm/simulators/linalg/ExtractParallelGridInformationToISTL.cpp opm/simulators/linalg/FlexibleSolver1.cpp opm/simulators/linalg/FlexibleSolver2.cpp @@ -409,8 +409,6 @@ list (APPEND TEST_DATA_FILES # originally generated with the command: # find opm -name '*.h*' -a ! -name '*-pch.hpp' -printf '\t%p\n' | sort list (APPEND PUBLIC_HEADER_FILES - ebos/collecttoiorank.hh - ebos/collecttoiorank_impl.hh ebos/ebos.hh ebos/eclbaseaquifermodel.hh ebos/eclbasevanguard.hh @@ -447,6 +445,8 @@ list (APPEND PUBLIC_HEADER_FILES opm/simulators/flow/BlackoilModel.hpp opm/simulators/flow/BlackoilModelNldd.hpp opm/simulators/flow/BlackoilModelParameters.hpp + opm/simulators/flow/CollectDataOnIORank.hpp + opm/simulators/flow/CollectDataOnIORank_impl.hpp opm/simulators/flow/ConvergenceOutputConfiguration.hpp opm/simulators/flow/countGlobalCells.hpp opm/simulators/flow/DummyGradientCalculator.hpp diff --git a/ebos/damariswriter.hh b/ebos/damariswriter.hh index ed918199e..20ce8a527 100644 --- a/ebos/damariswriter.hh +++ b/ebos/damariswriter.hh @@ -167,7 +167,7 @@ public: numElements_ = std::distance(interior_elements.begin(), interior_elements.end()); this->elements_rank_offsets_.resize(nranks_) ; - this->damarisOutputModule_ = std::make_unique>(simulator, this->collectToIORank_); + this->damarisOutputModule_ = std::make_unique>(simulator, this->collectOnIORank_); } /*! @@ -249,9 +249,9 @@ private: { // GLOBAL_CELL_INDEX is used to reorder variable data when writing to disk // This is enabled using select-file="GLOBAL_CELL_INDEX" in the XML tag - if (this->collectToIORank_.isParallel()) { + if (this->collectOnIORank_.isParallel()) { const std::vector& local_to_global = - this->collectToIORank_.localIdxToGlobalIdxMapping(); + this->collectOnIORank_.localIdxToGlobalIdxMapping(); dam_err_ = DamarisOutput::write("GLOBAL_CELL_INDEX", rank_, local_to_global.data()); } else { std::vector local_to_global_filled ; @@ -365,7 +365,7 @@ private: const auto& gridView = simulator_.vanguard().gridView(); const int num_interior = detail:: countLocalInteriorCellsGridView(gridView); - const bool log = this->collectToIORank_.isIORank(); + const bool log = this->collectOnIORank_.isIORank(); damarisOutputModule_->allocBuffers(num_interior, reportStepNum, isSubStep, log, /*isRestart*/ false); diff --git a/ebos/eclgenericwriter.hh b/ebos/eclgenericwriter.hh index 29d2cd58c..689e806cb 100644 --- a/ebos/eclgenericwriter.hh +++ b/ebos/eclgenericwriter.hh @@ -28,14 +28,13 @@ #ifndef EWOMS_ECL_GENERIC_WRITER_HH #define EWOMS_ECL_GENERIC_WRITER_HH -#include #include #include +#include #include -#include #include #include #include @@ -67,7 +66,7 @@ class EclGenericWriter { using CartesianIndexMapper = Dune::CartesianIndexMapper; using EquilCartesianIndexMapper = Dune::CartesianIndexMapper; - using CollectDataToIORankType = CollectDataToIORank; + using CollectDataOnIORankType = CollectDataOnIORank; using TransmissibilityType = EclTransmissibility; public: @@ -108,9 +107,9 @@ public: return outputNnc_; } - const CollectDataToIORankType& collectToIORank() const + const CollectDataOnIORankType& collectOnIORank() const { - return collectToIORank_; + return collectOnIORank_; } protected: @@ -151,7 +150,7 @@ protected: SummaryState& summaryState, UDQState& udqState); - CollectDataToIORankType collectToIORank_; + CollectDataOnIORankType collectOnIORank_; const Grid& grid_; const GridView& gridView_; const Schedule& schedule_; diff --git a/ebos/eclgenericwriter_impl.hh b/ebos/eclgenericwriter_impl.hh index c4503f83a..22366f863 100644 --- a/ebos/eclgenericwriter_impl.hh +++ b/ebos/eclgenericwriter_impl.hh @@ -197,7 +197,7 @@ EclGenericWriter(const Schedule& schedule, const Dune::CartesianIndexMapper* equilCartMapper, bool enableAsyncOutput, bool enableEsmry ) - : collectToIORank_(grid, + : collectOnIORank_(grid, equilGrid, gridView, cartMapper, @@ -212,7 +212,7 @@ EclGenericWriter(const Schedule& schedule, , equilCartMapper_(equilCartMapper) , equilGrid_ (equilGrid) { - if (this->collectToIORank_.isIORank()) { + if (this->collectOnIORank_.isIORank()) { this->eclIO_ = std::make_unique (this->eclState_, UgGridHelpers::createEclipseGrid(*equilGrid, eclState_.getInputGrid()), @@ -222,7 +222,7 @@ EclGenericWriter(const Schedule& schedule, // create output thread if enabled and rank is I/O rank // async output is enabled by default if pthread are enabled int numWorkerThreads = 0; - if (enableAsyncOutput && collectToIORank_.isIORank()) { + if (enableAsyncOutput && collectOnIORank_.isIORank()) { numWorkerThreads = 1; } @@ -241,10 +241,10 @@ template:: writeInit(const std::function& map) { - if (collectToIORank_.isIORank()) { + if (collectOnIORank_.isIORank()) { std::map> integerVectors; - if (collectToIORank_.isParallel()) { - integerVectors.emplace("MPI_RANK", collectToIORank_.globalRanks()); + if (collectOnIORank_.isParallel()) { + integerVectors.emplace("MPI_RANK", collectOnIORank_.globalRanks()); } auto cartMap = cartesianToCompressed(equilGrid_->size(0), UgGridHelpers::globalCell(*equilGrid_)); @@ -255,7 +255,7 @@ writeInit(const std::function& map) } #if HAVE_MPI - if (collectToIORank_.isParallel()) { + if (collectOnIORank_.isParallel()) { const auto& comm = grid_.comm(); Opm::Parallel::MpiSerializer ser(comm); ser.broadcast(outputNnc_); @@ -524,21 +524,21 @@ doWriteOutput(const int reportStepNum, bool isFloresn, std::array, 3>&& floresn) { - const auto isParallel = this->collectToIORank_.isParallel(); - const bool needsReordering = this->collectToIORank_.doesNeedReordering(); + const auto isParallel = this->collectOnIORank_.isParallel(); + const bool needsReordering = this->collectOnIORank_.doesNeedReordering(); RestartValue restartValue { (isParallel || needsReordering) - ? this->collectToIORank_.globalCellData() + ? this->collectOnIORank_.globalCellData() : std::move(localCellData), - isParallel ? this->collectToIORank_.globalWellData() + isParallel ? this->collectOnIORank_.globalWellData() : std::move(localWellData), - isParallel ? this->collectToIORank_.globalGroupAndNetworkData() + isParallel ? this->collectOnIORank_.globalGroupAndNetworkData() : std::move(localGroupAndNetworkData), - isParallel ? this->collectToIORank_.globalAquiferData() + isParallel ? this->collectOnIORank_.globalAquiferData() : std::move(localAquiferData) }; @@ -554,7 +554,7 @@ doWriteOutput(const int reportStepNum, // Add nnc flows and flores. if (isFlowsn) { - const auto flowsn_global = isParallel ? this->collectToIORank_.globalFlowsn() : std::move(flowsn); + const auto flowsn_global = isParallel ? this->collectOnIORank_.globalFlowsn() : std::move(flowsn); for (const auto& flows : flowsn_global) { if (flows.name.empty()) continue; @@ -566,7 +566,7 @@ doWriteOutput(const int reportStepNum, } } if (isFloresn) { - const auto floresn_global = isParallel ? this->collectToIORank_.globalFloresn() : std::move(floresn); + const auto floresn_global = isParallel ? this->collectOnIORank_.globalFloresn() : std::move(floresn); for (const auto& flores : floresn_global) { if (flores.name.empty()) { continue; @@ -579,7 +579,7 @@ doWriteOutput(const int reportStepNum, // step to disk auto eclWriteTasklet = std::make_shared( actionState, - isParallel ? this->collectToIORank_.globalWellTestState() : std::move(localWTestState), + isParallel ? this->collectOnIORank_.globalWellTestState() : std::move(localWTestState), summaryState, udqState, *this->eclIO_, reportStepNum, isSubStep, curTime, std::move(restartValue), doublePrecision); @@ -609,23 +609,23 @@ evalSummary(const int reportStepNum, SummaryState& summaryState, UDQState& udqState) { - if (collectToIORank_.isIORank()) { + if (collectOnIORank_.isIORank()) { const auto& summary = eclIO_->summary(); - const auto& wellData = this->collectToIORank_.isParallel() - ? this->collectToIORank_.globalWellData() + const auto& wellData = this->collectOnIORank_.isParallel() + ? this->collectOnIORank_.globalWellData() : localWellData; - const auto& wbpData = this->collectToIORank_.isParallel() - ? this->collectToIORank_.globalWBPData() + const auto& wbpData = this->collectOnIORank_.isParallel() + ? this->collectOnIORank_.globalWBPData() : localWBPData; - const auto& groupAndNetworkData = this->collectToIORank_.isParallel() - ? this->collectToIORank_.globalGroupAndNetworkData() + const auto& groupAndNetworkData = this->collectOnIORank_.isParallel() + ? this->collectOnIORank_.globalGroupAndNetworkData() : localGroupAndNetworkData; - const auto& aquiferData = this->collectToIORank_.isParallel() - ? this->collectToIORank_.globalAquiferData() + const auto& aquiferData = this->collectOnIORank_.isParallel() + ? this->collectOnIORank_.globalAquiferData() : localAquiferData; summary.eval(summaryState, @@ -658,7 +658,7 @@ evalSummary(const int reportStepNum, } #if HAVE_MPI - if (collectToIORank_.isParallel()) { + if (collectOnIORank_.isParallel()) { Parallel::MpiSerializer ser(grid_.comm()); ser.append(summaryState); } diff --git a/ebos/eclwriter.hh b/ebos/eclwriter.hh index 545910356..1969200b9 100644 --- a/ebos/eclwriter.hh +++ b/ebos/eclwriter.hh @@ -30,30 +30,28 @@ #include -#include #include #include +#include + #include #include +#include +#include #include #include #include -#include #include - -#include +#include +#include #include #include #include - -// #include - - namespace Opm::Properties { template @@ -157,7 +155,7 @@ public: , simulator_(simulator) { this->outputModule_ = std::make_unique> - (simulator, this->collectToIORank_); + (simulator, this->collectOnIORank_); rank_ = simulator_.vanguard().grid().comm().rank() ; } @@ -218,10 +216,10 @@ public: this->captureLocalFluxData(); } - if (this->collectToIORank_.isParallel()) { + if (this->collectOnIORank_.isParallel()) { OPM_BEGIN_PARALLEL_TRY_CATCH() - this->collectToIORank_.collect({}, + this->collectOnIORank_.collect({}, outputModule_->getBlockData(), localWellData, localWBP, @@ -232,8 +230,8 @@ public: {}, {}); - if (this->collectToIORank_.isIORank()) { - auto& iregFlows = this->collectToIORank_.globalInterRegFlows(); + if (this->collectOnIORank_.isIORank()) { + auto& iregFlows = this->collectOnIORank_.globalInterRegFlows(); if (! iregFlows.readIsConsistent()) { throw std::runtime_error { @@ -259,7 +257,7 @@ public: inplace = outputModule_->calc_inplace(miscSummaryData, regionData, simulator_.gridView().comm()); - if (this->collectToIORank_.isIORank()){ + if (this->collectOnIORank_.isIORank()){ inplace_ = inplace; } } @@ -293,12 +291,12 @@ public: { OPM_TIMEBLOCK(evalSummary); - const auto& blockData = this->collectToIORank_.isParallel() - ? this->collectToIORank_.globalBlockData() + const auto& blockData = this->collectOnIORank_.isParallel() + ? this->collectOnIORank_.globalBlockData() : this->outputModule_->getBlockData(); - const auto& interRegFlows = this->collectToIORank_.isParallel() - ? this->collectToIORank_.globalInterRegFlows() + const auto& interRegFlows = this->collectOnIORank_.isParallel() + ? this->collectOnIORank_.globalInterRegFlows() : this->outputModule_->getInterRegFlows(); this->evalSummary(reportStepNum, @@ -354,7 +352,7 @@ public: inplace = outputModule_->calc_inplace(miscSummaryData, regionData, simulator_.gridView().comm()); - if (this->collectToIORank_.isIORank()){ + if (this->collectOnIORank_.isIORank()){ inplace_ = inplace; outputModule_->outputFipAndResvLog(inplace_, 0, 0.0, start_time, @@ -390,7 +388,7 @@ public: auto rstep = timer.reportStepNum(); - if ((rstep > 0) && (this->collectToIORank_.isIORank())){ + if ((rstep > 0) && (this->collectOnIORank_.isIORank())){ outputModule_->outputFipAndResvLog(inplace_, rstep, timer.simulationTimeElapsed(), timer.currentDateTime(), false, simulator_.gridView().comm()); @@ -413,15 +411,15 @@ public: this->outputModule_->addRftDataToWells(localWellData, reportStepNum); } - if (this->collectToIORank_.isParallel() || - this->collectToIORank_.doesNeedReordering()) + if (this->collectOnIORank_.isParallel() || + this->collectOnIORank_.doesNeedReordering()) { // Note: We don't need WBP (well-block averaged pressures) or // inter-region flow rate values in order to create restart file // output. There's consequently no need to collect those // properties on the I/O rank. - this->collectToIORank_.collect(localCellData, + this->collectOnIORank_.collect(localCellData, this->outputModule_->getBlockData(), localWellData, /* wbpData = */ {}, @@ -431,14 +429,14 @@ public: /* interRegFlows = */ {}, flowsn, floresn); - if (this->collectToIORank_.isIORank()) { - this->outputModule_->assignGlobalFieldsToSolution(this->collectToIORank_.globalCellData()); + if (this->collectOnIORank_.isIORank()) { + this->outputModule_->assignGlobalFieldsToSolution(this->collectOnIORank_.globalCellData()); } } else { this->outputModule_->assignGlobalFieldsToSolution(localCellData); } - if (this->collectToIORank_.isIORank()) { + if (this->collectOnIORank_.isIORank()) { const Scalar curTime = simulator_.time() + simulator_.timeStepSize(); const Scalar nextStepSize = simulator_.problem().nextTimeStepSize(); @@ -509,7 +507,7 @@ public: auto restartValues = loadParallelRestart(this->eclIO_.get(), actionState, summaryState, solutionKeys, extraKeys, gridView.grid().comm()); for (unsigned elemIdx = 0; elemIdx < numElements; ++elemIdx) { - unsigned globalIdx = this->collectToIORank_.localIdxToGlobalIdx(elemIdx); + unsigned globalIdx = this->collectOnIORank_.localIdxToGlobalIdx(elemIdx); outputModule_->setRestart(restartValues.solution, elemIdx, globalIdx); } @@ -518,7 +516,7 @@ public: const auto& tracer_name = tracer_model.fname(tracer_index); const auto& tracer_solution = restartValues.solution.template data(tracer_name); for (unsigned elemIdx = 0; elemIdx < numElements; ++elemIdx) { - unsigned globalIdx = this->collectToIORank_.localIdxToGlobalIdx(elemIdx); + unsigned globalIdx = this->collectOnIORank_.localIdxToGlobalIdx(elemIdx); tracer_model.setTracerConcentration(tracer_index, globalIdx, tracer_solution[globalIdx]); } } @@ -586,7 +584,7 @@ private: } const auto& gridView = simulator_.vanguard().gridView(); - const bool log = this->collectToIORank_.isIORank(); + const bool log = this->collectOnIORank_.isIORank(); const int num_interior = detail:: countLocalInteriorCellsGridView(gridView); diff --git a/flow/flow_blackoil_alugrid.cpp b/flow/flow_blackoil_alugrid.cpp index e0a98fa5d..ff71e74a7 100644 --- a/flow/flow_blackoil_alugrid.cpp +++ b/flow/flow_blackoil_alugrid.cpp @@ -27,7 +27,6 @@ #include // these are not explicitly instanced in library -#include #include #include #include @@ -35,6 +34,7 @@ #include #include #include +#include #include namespace Opm { diff --git a/flow/flow_blackoil_polyhedralgrid.cpp b/flow/flow_blackoil_polyhedralgrid.cpp index 994856acc..96d717aa0 100644 --- a/flow/flow_blackoil_polyhedralgrid.cpp +++ b/flow/flow_blackoil_polyhedralgrid.cpp @@ -26,13 +26,13 @@ #include // these are not explicitly instanced in library -#include #include #include #include #include #include #include +#include #include namespace Opm { diff --git a/opm/simulators/flow/BlackoilModel.hpp b/opm/simulators/flow/BlackoilModel.hpp index 4934a0210..956457449 100644 --- a/opm/simulators/flow/BlackoilModel.hpp +++ b/opm/simulators/flow/BlackoilModel.hpp @@ -234,7 +234,7 @@ namespace Opm { , phaseUsage_(phaseUsageFromDeck(eclState())) , param_( param ) , well_model_ (well_model) - , rst_conv_(simulator_.problem().eclWriter()->collectToIORank().localIdxToGlobalIdxMapping(), + , rst_conv_(simulator_.problem().eclWriter()->collectOnIORank().localIdxToGlobalIdxMapping(), grid_.comm()) , terminal_output_ (terminal_output) , current_relaxation_(1.0) diff --git a/ebos/collecttoiorank.cc b/opm/simulators/flow/CollectDataOnIORank.cpp similarity index 90% rename from ebos/collecttoiorank.cc rename to opm/simulators/flow/CollectDataOnIORank.cpp index 5532716c9..14255c775 100644 --- a/ebos/collecttoiorank.cc +++ b/opm/simulators/flow/CollectDataOnIORank.cpp @@ -22,8 +22,7 @@ */ #include -#include -#include +#include #include @@ -35,15 +34,15 @@ namespace Opm { -template class CollectDataToIORank>>; #if HAVE_DUNE_FEM -template class CollectDataToIORank>>>; -template class CollectDataToIORank +#ifndef OPM_COLLECT_DATA_ON_IO_RANK_HPP +#define OPM_COLLECT_DATA_ON_IO_RANK_HPP #include +#include + #include #include #include @@ -52,7 +52,7 @@ template class CartesianIndexMapper; namespace Opm { template -class CollectDataToIORank +class CollectDataOnIORank { public: using CollectiveCommunication = typename Grid::CollectiveCommunication; @@ -67,7 +67,7 @@ public: static const bool needsReordering = !std::is_same::value; - CollectDataToIORank(const Grid& grid, + CollectDataOnIORank(const Grid& grid, const EquilGrid* equilGrid, const GridView& gridView, const Dune::CartesianIndexMapper& cartMapper, @@ -171,4 +171,4 @@ protected: } // end namespace Opm -#endif +#endif // OPM_COLLECT_DATA_ON_IO_RANK_HPP diff --git a/ebos/collecttoiorank_impl.hh b/opm/simulators/flow/CollectDataOnIORank_impl.hpp similarity index 98% rename from ebos/collecttoiorank_impl.hh rename to opm/simulators/flow/CollectDataOnIORank_impl.hpp index a398b914a..b91e79b1e 100644 --- a/ebos/collecttoiorank_impl.hh +++ b/opm/simulators/flow/CollectDataOnIORank_impl.hpp @@ -21,18 +21,18 @@ copyright holders. */ -#ifndef EWOMS_COLLECT_TO_IO_RANK_IMPL_HH -#define EWOMS_COLLECT_TO_IO_RANK_IMPL_HH +#ifndef OPM_COLLECT_DATA_ON_IO_RANK_IMPL_HPP +#define OPM_COLLECT_DATA_ON_IO_RANK_IMPL_HPP -#include - -#include +#include #include #include #include #include +#include + #include #include #include @@ -842,8 +842,8 @@ public: }; template -CollectDataToIORank:: -CollectDataToIORank(const Grid& grid, const EquilGrid* equilGrid, +CollectDataOnIORank:: +CollectDataOnIORank(const Grid& grid, const EquilGrid* equilGrid, const GridView& localGridView, const Dune::CartesianIndexMapper& cartMapper, const Dune::CartesianIndexMapper* equilCartMapper, @@ -965,7 +965,7 @@ CollectDataToIORank(const Grid& grid, const EquilGrid* equilGrid, } template -void CollectDataToIORank:: +void CollectDataOnIORank:: collect(const data::Solution& localCellData, const std::map, double>& localBlockData, const data::Wells& localWellData, @@ -1087,7 +1087,7 @@ collect(const data::Solution& localCellData, } template -int CollectDataToIORank:: +int CollectDataOnIORank:: localIdxToGlobalIdx(unsigned localIdx) const { if (!isParallel()) { @@ -1106,7 +1106,7 @@ localIdxToGlobalIdx(unsigned localIdx) const } template -bool CollectDataToIORank:: +bool CollectDataOnIORank:: isCartIdxOnThisRank(int cartIdx) const { if (! this->isParallel()) { @@ -1124,4 +1124,5 @@ isCartIdxOnThisRank(int cartIdx) const } } // end namespace Opm -#endif + +#endif // OPM_COLLECT_DATA_ON_IO_RANK_IMPL_HPP