diff --git a/benchmarks/upscale_relperm_benchmark.cpp b/benchmarks/upscale_relperm_benchmark.cpp index c4c2a63..f2ab06b 100644 --- a/benchmarks/upscale_relperm_benchmark.cpp +++ b/benchmarks/upscale_relperm_benchmark.cpp @@ -1375,72 +1375,7 @@ try clock_t finish_upscale_wallclock = clock(); timeused_upscale_wallclock = (double(finish_upscale_wallclock)-double(start_upscale_wallclock))/CLOCKS_PER_SEC; -#ifdef HAVE_MPI - /* Step 8b: Transfer all computed data to master node. - Master node should post a receive for all values missing, - other nodes should post a send for all the values they have. - */ - MPI_Barrier(MPI_COMM_WORLD); // Not strictly necessary. - if (helper.isMaster) { - // Loop over all values, receive data and put into local data structure - for (int idx=0; idx < points; ++idx) { - if (node_vs_pressurepoint[idx] != 0) { - // Receive data - if (helper.upscaleBothPhases) { - std::vector recvbuffer(2+2*helper.tensorElementCount); - MPI_Recv(recvbuffer.data(), recvbuffer.size(), MPI_DOUBLE, - node_vs_pressurepoint[idx], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - // Put received data into correct place. - helper.WaterSaturation[(int)recvbuffer[0]] = recvbuffer[1]; - for (int voigtIdx=0; voigtIdx < helper.tensorElementCount; ++voigtIdx) { - PhasePerm[(int)recvbuffer[0]][voigtIdx] = recvbuffer[2+voigtIdx]; - } - for (int voigtIdx=0; voigtIdx < helper.tensorElementCount; ++voigtIdx) { - Phase2Perm[(int)recvbuffer[0]][voigtIdx] = recvbuffer[2+helper.tensorElementCount+voigtIdx]; - } - } - else { - std::vector recvbuffer(2+helper.tensorElementCount); - MPI_Recv(recvbuffer.data(), recvbuffer.size(), MPI_DOUBLE, - node_vs_pressurepoint[idx], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - // Put received data into correct place. - helper.WaterSaturation[(int)recvbuffer[0]] = recvbuffer[1]; - for (int voigtIdx=0; voigtIdx < helper.tensorElementCount; ++voigtIdx) { - PhasePerm[(int)recvbuffer[0]][voigtIdx] = recvbuffer[2+voigtIdx]; - } - } - } - } - } - else { - for (int idx=0; idx < points; ++idx) { - if (node_vs_pressurepoint[idx] == mpi_rank) { - // Pack and send data. C-style. - if (helper.upscaleBothPhases) { - std::vector sendbuffer(2+2*helper.tensorElementCount); - sendbuffer[0] = (double)idx; - sendbuffer[1] = helper.WaterSaturation[idx]; - for (int voigtIdx=0; voigtIdx < helper.tensorElementCount; ++voigtIdx) { - sendbuffer[2+voigtIdx] = PhasePerm[idx][voigtIdx]; - } - for (int voigtIdx=0; voigtIdx < helper.tensorElementCount; ++voigtIdx) { - sendbuffer[2+helper.tensorElementCount+voigtIdx] = Phase2Perm[idx][voigtIdx]; - } - MPI_Send(sendbuffer.data(), sendbuffer.size(), MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); - } - else { - std::vector sendbuffer(2+helper.tensorElementCount); - sendbuffer[0] = (double)idx; - sendbuffer[1] = helper.WaterSaturation[idx]; - for (int voigtIdx=0; voigtIdx < helper.tensorElementCount; ++voigtIdx) { - sendbuffer[2+voigtIdx] = PhasePerm[idx][voigtIdx]; - } - MPI_Send(sendbuffer.data(), sendbuffer.size(), MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); - } - } - } - } -#endif + helper.collectResults(); // Average time pr. upscaling point: #ifdef HAVE_MPI diff --git a/examples/upscale_relperm.cpp b/examples/upscale_relperm.cpp index 47ca33a..8bbc316 100644 --- a/examples/upscale_relperm.cpp +++ b/examples/upscale_relperm.cpp @@ -1513,72 +1513,7 @@ try clock_t finish_upscale_wallclock = clock(); timeused_upscale_wallclock = (double(finish_upscale_wallclock)-double(start_upscale_wallclock))/CLOCKS_PER_SEC; -#ifdef HAVE_MPI - /* Step 8b: Transfer all computed data to master node. - Master node should post a receive for all values missing, - other nodes should post a send for all the values they have. - */ - MPI_Barrier(MPI_COMM_WORLD); // Not strictly necessary. - if (isMaster) { - // Loop over all values, receive data and put into local data structure - for (int idx=0; idx < points; ++idx) { - if (node_vs_pressurepoint[idx] != 0) { - // Receive data - if (upscaleBothPhases) { - std::vector recvbuffer(2+2*tensorElementCount); - MPI_Recv(recvbuffer.data(), recvbuffer.size(), MPI_DOUBLE, - node_vs_pressurepoint[idx], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - // Put received data into correct place. - WaterSaturation[(int)recvbuffer[0]] = recvbuffer[1]; - for (int voigtIdx=0; voigtIdx < tensorElementCount; ++voigtIdx) { - PhasePerm[(int)recvbuffer[0]][voigtIdx] = recvbuffer[2+voigtIdx]; - } - for (int voigtIdx=0; voigtIdx < tensorElementCount; ++voigtIdx) { - Phase2Perm[(int)recvbuffer[0]][voigtIdx] = recvbuffer[2+tensorElementCount+voigtIdx]; - } - } - else { - std::vector recvbuffer(2+tensorElementCount); - MPI_Recv(recvbuffer.data(), recvbuffer.size(), MPI_DOUBLE, - node_vs_pressurepoint[idx], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - // Put received data into correct place. - WaterSaturation[(int)recvbuffer[0]] = recvbuffer[1]; - for (int voigtIdx=0; voigtIdx < tensorElementCount; ++voigtIdx) { - PhasePerm[(int)recvbuffer[0]][voigtIdx] = recvbuffer[2+voigtIdx]; - } - } - } - } - } - else { - for (int idx=0; idx < points; ++idx) { - if (node_vs_pressurepoint[idx] == mpi_rank) { - // Pack and send data. C-style. - if (upscaleBothPhases) { - std::vector sendbuffer(2+2*tensorElementCount); - sendbuffer[0] = (double)idx; - sendbuffer[1] = WaterSaturation[idx]; - for (int voigtIdx=0; voigtIdx < tensorElementCount; ++voigtIdx) { - sendbuffer[2+voigtIdx] = PhasePerm[idx][voigtIdx]; - } - for (int voigtIdx=0; voigtIdx < tensorElementCount; ++voigtIdx) { - sendbuffer[2+tensorElementCount+voigtIdx] = Phase2Perm[idx][voigtIdx]; - } - MPI_Send(sendbuffer.data(), sendbuffer.size(), MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); - } - else { - std::vector sendbuffer(2+tensorElementCount); - sendbuffer[0] = (double)idx; - sendbuffer[1] = WaterSaturation[idx]; - for (int voigtIdx=0; voigtIdx < tensorElementCount; ++voigtIdx) { - sendbuffer[2+voigtIdx] = PhasePerm[idx][voigtIdx]; - } - MPI_Send(sendbuffer.data(), sendbuffer.size(), MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); - } - } - } - } -#endif + helper.collectResults(); // Average time pr. upscaling point: #ifdef HAVE_MPI diff --git a/opm/upscaling/RelPermUtils.cpp b/opm/upscaling/RelPermUtils.cpp index 1f17958..2019619 100644 --- a/opm/upscaling/RelPermUtils.cpp +++ b/opm/upscaling/RelPermUtils.cpp @@ -60,4 +60,74 @@ void setVoigtValue(SinglePhaseUpscaler::permtensor_t& K, int voigt_idx, double v K.data()[voigt_idx_tab[voigt_idx]] = val; } +void RelPermUpscaleHelper::collectResults() +{ +#ifdef HAVE_MPI + /* Step 8b: Transfer all computed data to master node. + Master node should post a receive for all values missing, + other nodes should post a send for all the values they have. + */ + MPI_Barrier(MPI_COMM_WORLD); // Not strictly necessary. + if (isMaster) { + // Loop over all values, receive data and put into local data structure + for (int idx=0; idx < points; ++idx) { + if (node_vs_pressurepoint[idx] != 0) { + // Receive data + if (upscaleBothPhases) { + std::vector recvbuffer(2+2*tensorElementCount); + MPI_Recv(recvbuffer.data(), recvbuffer.size(), MPI_DOUBLE, + node_vs_pressurepoint[idx], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); + // Put received data into correct place. + WaterSaturation[(int)recvbuffer[0]] = recvbuffer[1]; + for (int voigtIdx=0; voigtIdx < tensorElementCount; ++voigtIdx) { + PhasePerm[(int)recvbuffer[0]][voigtIdx] = recvbuffer[2+voigtIdx]; + } + for (int voigtIdx=0; voigtIdx < tensorElementCount; ++voigtIdx) { + Phase2Perm[(int)recvbuffer[0]][voigtIdx] = recvbuffer[2+tensorElementCount+voigtIdx]; + } + } + else { + std::vector recvbuffer(2+tensorElementCount); + MPI_Recv(recvbuffer.data(), recvbuffer.size(), MPI_DOUBLE, + node_vs_pressurepoint[idx], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); + // Put received data into correct place. + WaterSaturation[(int)recvbuffer[0]] = recvbuffer[1]; + for (int voigtIdx=0; voigtIdx < tensorElementCount; ++voigtIdx) { + PhasePerm[(int)recvbuffer[0]][voigtIdx] = recvbuffer[2+voigtIdx]; + } + } + } + } + } + else { + for (int idx=0; idx < points; ++idx) { + if (node_vs_pressurepoint[idx] == mpi_rank) { + // Pack and send data. C-style. + if (upscaleBothPhases) { + std::vector sendbuffer(2+2*tensorElementCount); + sendbuffer[0] = (double)idx; + sendbuffer[1] = WaterSaturation[idx]; + for (int voigtIdx=0; voigtIdx < tensorElementCount; ++voigtIdx) { + sendbuffer[2+voigtIdx] = PhasePerm[idx][voigtIdx]; + } + for (int voigtIdx=0; voigtIdx < tensorElementCount; ++voigtIdx) { + sendbuffer[2+tensorElementCount+voigtIdx] = Phase2Perm[idx][voigtIdx]; + } + MPI_Send(sendbuffer.data(), sendbuffer.size(), MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); + } + else { + std::vector sendbuffer(2+tensorElementCount); + sendbuffer[0] = (double)idx; + sendbuffer[1] = WaterSaturation[idx]; + for (int voigtIdx=0; voigtIdx < tensorElementCount; ++voigtIdx) { + sendbuffer[2+voigtIdx] = PhasePerm[idx][voigtIdx]; + } + MPI_Send(sendbuffer.data(), sendbuffer.size(), MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); + } + } + } + } +#endif +} + }