#ifndef COMMUNICATION_H_INC #define COMMUNICATION_H_INC #include "common/MPI.h" #include "common/Utilities.h" #include "common/Array.h" #include // ********** COMMUNICTION ************************************** /* //.............................................................. // Communication helper routines for MPI //.............................................................. */ using namespace std; /*! * @brief Rank info structure * @details Structure used to hold the ranks for the current process and it's neighbors */ struct RankInfoStruct { int nx; //!< The number of processors in the x direction int ny; //!< The number of processors in the y direction int nz; //!< The number of processors in the z direction int ix; //!< The index of the current process in the x direction int jy; //!< The index of the current process in the y direction int kz; //!< The index of the current process in the z direction int rank[3][3][3]; //!< The rank for the neighbor [i][j][k] RankInfoStruct(); RankInfoStruct( int rank, int nprocx, int nprocy, int nprocz ); int getRankForBlock( int i, int j, int k ) const; }; //! Redistribute domain data (dst may be smaller than the src) template Array redistribute( const RankInfoStruct& src_rank, const Array& src_data, const RankInfoStruct& dst_rank, std::array dst_size, const Utilities::MPI& comm ); /*! * @brief Communicate halo * @details Fill the halo cells in an array from the neighboring processes */ template class fillHalo { public: /*! * @brief Default constructor * @param[in] comm Communicator to use * @param[in] info Rank and neighbor rank info * @param[in] n Number of local cells * @param[in] ng Number of ghost cells * @param[in] tag Initial tag to use for the communication (we will require tag:tag+26) * @param[in] depth Maximum depth to support * @param[in] fill Fill {faces,edges,corners} * @param[in] periodic Periodic dimensions */ fillHalo( const Utilities::MPI& comm, const RankInfoStruct& info, std::array n, std::array ng, int tag, int depth, std::array fill = {true,true,true}, std::array periodic = {true,true,true} ); //! Destructor ~fillHalo( ); fillHalo() = delete; fillHalo(const fillHalo&) = delete; fillHalo& operator=(const fillHalo&) = delete; /*! * @brief Communicate the halos * @param[in] array The array on which we fill the halos */ void fill( Array& array ); /*! * @brief Copy data from the src array to the dst array * @param[in] src The src array with or without halos * @param[in] dst The dst array with or without halos */ template void copy( const Array& src, Array& dst ); private: Utilities::MPI comm; RankInfoStruct info; std::array n, ng; int depth; bool fill_pattern[3][3][3]; int tag[3][3][3]; int N_send_recv[3][3][3]; TYPE *mem; TYPE *send[3][3][3], *recv[3][3][3]; MPI_Request send_req[3][3][3], recv_req[3][3][3]; void pack( const Array& array, int i, int j, int k, TYPE *buffer ); void unpack( Array& array, int i, int j, int k, const TYPE *buffer ); }; //*************************************************************************************** inline void PackMeshData(const int *list, int count, double *sendbuf, double *data){ // Fill in the phase ID values from neighboring processors // This packs up the values that need to be sent from one processor to another int idx,n; for (idx=0; idx