#ifndef COMMUNICATION_H_INC #define COMMUNICATION_H_INC #include "common/MPI_Helpers.h" #include "common/Utilities.h" #include "common/Array.h" #include // ********** COMMUNICTION ************************************** /* //.............................................................. // Communication helper routines for MPI //.............................................................. */ using namespace std; /*! * @brief Rank info structure * @details Structure used to hold the ranks for the current process and it's neighbors */ struct RankInfoStruct { int nx; //!< The number of processors in the x direction int ny; //!< The number of processors in the y direction int nz; //!< The number of processors in the z direction int ix; //!< The index of the current process in the x direction int jy; //!< The index of the current process in the y direction int kz; //!< The index of the current process in the z direction int rank[3][3][3]; //!< The rank for the neighbor [i][j][k] RankInfoStruct(); RankInfoStruct( int rank, int nprocx, int nprocy, int nprocz ); }; /*! * @brief Communicate halo * @details Fill the halo cells in an array from the neighboring processes */ template class fillHalo { public: /*! * @brief Default constructor * @param[in] info Rank and neighbor rank info * @param[in] n Number of local cells * @param[in] ng Number of ghost cells * @param[in] tag Initial tag to use for the communication (we will require tag:tag+26) * @param[in] depth Maximum depth to support * @param[in] fill Fill {faces,edges,corners} * @param[in] periodic Periodic dimensions */ fillHalo( MPI_Comm comm, const RankInfoStruct& info, std::array n, std::array ng, int tag, int depth, std::array fill = {true,true,true}, std::array periodic = {true,true,true} ); //! Destructor ~fillHalo( ); /*! * @brief Communicate the halos * @param[in] array The array on which we fill the halos */ void fill( Array& array ); /*! * @brief Copy data from the src array to the dst array * @param[in] src The src array with or without halos * @param[in] dst The dst array with or without halos */ template void copy( const Array& src, Array& dst ); private: MPI_Comm comm; RankInfoStruct info; std::array n, ng; int depth; bool fill_pattern[3][3][3]; int tag[3][3][3]; int N_send_recv[3][3][3]; TYPE *mem; TYPE *send[3][3][3], *recv[3][3][3]; MPI_Request send_req[3][3][3], recv_req[3][3][3]; size_t N_type; MPI_Datatype datatype; fillHalo(); // Private empty constructor fillHalo(const fillHalo&); // Private copy constructor fillHalo& operator=(const fillHalo&); // Private assignment operator void pack( const Array& array, int i, int j, int k, TYPE *buffer ); void unpack( Array& array, int i, int j, int k, const TYPE *buffer ); }; //*************************************************************************************** inline void PackMeshData(int *list, int count, double *sendbuf, double *data){ // Fill in the phase ID values from neighboring processors // This packs up the values that need to be sent from one processor to another int idx,n; for (idx=0; idx