refactored TestBubble.cpp

This commit is contained in:
James E McClure 2014-03-19 09:26:34 -04:00
parent 05bbbd3e5d
commit c3bfaddc37

View File

@ -642,43 +642,43 @@ int main(int argc, char **argv)
double *recvbuf_xy, *recvbuf_yz, *recvbuf_xz, *recvbuf_Xy, *recvbuf_Yz, *recvbuf_xZ; double *recvbuf_xy, *recvbuf_yz, *recvbuf_xz, *recvbuf_Xy, *recvbuf_Yz, *recvbuf_xZ;
double *recvbuf_xY, *recvbuf_yZ, *recvbuf_Xz, *recvbuf_XY, *recvbuf_YZ, *recvbuf_XZ; double *recvbuf_xY, *recvbuf_yZ, *recvbuf_Xz, *recvbuf_XY, *recvbuf_YZ, *recvbuf_XZ;
//...................................................................................... //......................................................................................
dvc_AllocateDeviceMemory((void **) &sendbuf_x, 5*sendCount_x*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_x, 5*sendCount_x*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_X, 5*sendCount_X*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_X, 5*sendCount_X*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_y, 5*sendCount_y*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_y, 5*sendCount_y*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_Y, 5*sendCount_Y*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_Y, 5*sendCount_Y*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_z, 5*sendCount_z*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_z, 5*sendCount_z*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_Z, 5*sendCount_Z*sizeof(double)); // Allocatevoid * memory AllocateDeviceMemory((void **) &sendbuf_Z, 5*sendCount_Z*sizeof(double)); // Allocatevoid * memory
dvc_AllocateDeviceMemory((void **) &sendbuf_xy, sendCount_xy*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_xy, sendCount_xy*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_xY, sendCount_xY*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_xY, sendCount_xY*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_Xy, sendCount_Xy*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_Xy, sendCount_Xy*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_XY, sendCount_XY*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_XY, sendCount_XY*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_xz, sendCount_xz*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_xz, sendCount_xz*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_xZ, sendCount_xZ*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_xZ, sendCount_xZ*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_Xz, sendCount_Xz*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_Xz, sendCount_Xz*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_XZ, sendCount_XZ*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_XZ, sendCount_XZ*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_yz, sendCount_yz*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_yz, sendCount_yz*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_yZ, sendCount_yZ*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_yZ, sendCount_yZ*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_Yz, sendCount_Yz*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_Yz, sendCount_Yz*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &sendbuf_YZ, sendCount_YZ*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &sendbuf_YZ, sendCount_YZ*sizeof(double)); // Allocate device memory
//...................................................................................... //......................................................................................
dvc_AllocateDeviceMemory((void **) &recvbuf_x, 5*recvCount_x*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_x, 5*recvCount_x*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_X, 5*recvCount_X*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_X, 5*recvCount_X*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_y, 5*recvCount_y*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_y, 5*recvCount_y*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_Y, 5*recvCount_Y*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_Y, 5*recvCount_Y*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_z, 5*recvCount_z*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_z, 5*recvCount_z*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_Z, 5*recvCount_Z*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_Z, 5*recvCount_Z*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_xy, recvCount_xy*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_xy, recvCount_xy*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_xY, recvCount_xY*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_xY, recvCount_xY*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_Xy, recvCount_Xy*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_Xy, recvCount_Xy*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_XY, recvCount_XY*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_XY, recvCount_XY*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_xz, recvCount_xz*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_xz, recvCount_xz*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_xZ, recvCount_xZ*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_xZ, recvCount_xZ*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_Xz, recvCount_Xz*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_Xz, recvCount_Xz*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_XZ, recvCount_XZ*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_XZ, recvCount_XZ*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_yz, recvCount_yz*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_yz, recvCount_yz*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_yZ, recvCount_yZ*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_yZ, recvCount_yZ*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_Yz, recvCount_Yz*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_Yz, recvCount_Yz*sizeof(double)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &recvbuf_YZ, recvCount_YZ*sizeof(double)); // Allocate device memory AllocateDeviceMemory((void **) &recvbuf_YZ, recvCount_YZ*sizeof(double)); // Allocate device memory
//...................................................................................... //......................................................................................
int *dvcSendList_x, *dvcSendList_y, *dvcSendList_z, *dvcSendList_X, *dvcSendList_Y, *dvcSendList_Z; int *dvcSendList_x, *dvcSendList_y, *dvcSendList_z, *dvcSendList_X, *dvcSendList_Y, *dvcSendList_Z;
int *dvcSendList_xy, *dvcSendList_yz, *dvcSendList_xz, *dvcSendList_Xy, *dvcSendList_Yz, *dvcSendList_xZ; int *dvcSendList_xy, *dvcSendList_yz, *dvcSendList_xz, *dvcSendList_Xy, *dvcSendList_Yz, *dvcSendList_xZ;
@ -688,83 +688,83 @@ int main(int argc, char **argv)
int *dvcRecvList_xy, *dvcRecvList_yz, *dvcRecvList_xz, *dvcRecvList_Xy, *dvcRecvList_Yz, *dvcRecvList_xZ; int *dvcRecvList_xy, *dvcRecvList_yz, *dvcRecvList_xz, *dvcRecvList_Xy, *dvcRecvList_Yz, *dvcRecvList_xZ;
int *dvcRecvList_xY, *dvcRecvList_yZ, *dvcRecvList_Xz, *dvcRecvList_XY, *dvcRecvList_YZ, *dvcRecvList_XZ; int *dvcRecvList_xY, *dvcRecvList_yZ, *dvcRecvList_Xz, *dvcRecvList_XY, *dvcRecvList_YZ, *dvcRecvList_XZ;
//...................................................................................... //......................................................................................
dvc_AllocateDeviceMemory((void **) &dvcSendList_x, sendCount_x*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_x, sendCount_x*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_X, sendCount_X*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_X, sendCount_X*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_y, sendCount_y*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_y, sendCount_y*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_Y, sendCount_Y*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_Y, sendCount_Y*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_z, sendCount_z*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_z, sendCount_z*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_Z, sendCount_Z*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_Z, sendCount_Z*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_xy, sendCount_xy*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_xy, sendCount_xy*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_xY, sendCount_xY*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_xY, sendCount_xY*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_Xy, sendCount_Xy*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_Xy, sendCount_Xy*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_XY, sendCount_XY*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_XY, sendCount_XY*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_xz, sendCount_xz*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_xz, sendCount_xz*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_xZ, sendCount_xZ*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_xZ, sendCount_xZ*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_Xz, sendCount_Xz*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_Xz, sendCount_Xz*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_XZ, sendCount_XZ*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_XZ, sendCount_XZ*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_yz, sendCount_yz*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_yz, sendCount_yz*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_yZ, sendCount_yZ*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_yZ, sendCount_yZ*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_Yz, sendCount_Yz*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_Yz, sendCount_Yz*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcSendList_YZ, sendCount_YZ*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcSendList_YZ, sendCount_YZ*sizeof(int)); // Allocate device memory
//...................................................................................... //......................................................................................
dvc_AllocateDeviceMemory((void **) &dvcRecvList_x, recvCount_x*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_x, recvCount_x*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_X, recvCount_X*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_X, recvCount_X*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_y, recvCount_y*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_y, recvCount_y*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_Y, recvCount_Y*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_Y, recvCount_Y*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_z, recvCount_z*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_z, recvCount_z*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_Z, recvCount_Z*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_Z, recvCount_Z*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_xy, recvCount_xy*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_xy, recvCount_xy*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_xY, recvCount_xY*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_xY, recvCount_xY*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_Xy, recvCount_Xy*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_Xy, recvCount_Xy*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_XY, recvCount_XY*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_XY, recvCount_XY*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_xz, recvCount_xz*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_xz, recvCount_xz*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_xZ, recvCount_xZ*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_xZ, recvCount_xZ*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_Xz, recvCount_Xz*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_Xz, recvCount_Xz*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_XZ, recvCount_XZ*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_XZ, recvCount_XZ*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_yz, recvCount_yz*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_yz, recvCount_yz*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_yZ, recvCount_yZ*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_yZ, recvCount_yZ*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_Yz, recvCount_Yz*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_Yz, recvCount_Yz*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory AllocateDeviceMemory((void **) &dvcRecvList_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory
//...................................................................................... //......................................................................................
MPI_Barrier(MPI_COMM_WORLD); MPI_Barrier(MPI_COMM_WORLD);
if (rank==0) printf ("Prepare to copy send/recv Lists to device \n"); if (rank==0) printf ("Prepare to copy send/recv Lists to device \n");
dvc_CopyToDevice(dvcSendList_x,sendList_x,sendCount_x*sizeof(int)); CopyToDevice(dvcSendList_x,sendList_x,sendCount_x*sizeof(int));
dvc_CopyToDevice(dvcSendList_X,sendList_X,sendCount_X*sizeof(int)); CopyToDevice(dvcSendList_X,sendList_X,sendCount_X*sizeof(int));
dvc_CopyToDevice(dvcSendList_y,sendList_y,sendCount_y*sizeof(int)); CopyToDevice(dvcSendList_y,sendList_y,sendCount_y*sizeof(int));
dvc_CopyToDevice(dvcSendList_Y,sendList_Y,sendCount_Y*sizeof(int)); CopyToDevice(dvcSendList_Y,sendList_Y,sendCount_Y*sizeof(int));
dvc_CopyToDevice(dvcSendList_z,sendList_z,sendCount_z*sizeof(int)); CopyToDevice(dvcSendList_z,sendList_z,sendCount_z*sizeof(int));
dvc_CopyToDevice(dvcSendList_Z,sendList_Z,sendCount_Z*sizeof(int)); CopyToDevice(dvcSendList_Z,sendList_Z,sendCount_Z*sizeof(int));
dvc_CopyToDevice(dvcSendList_xy,sendList_xy,sendCount_xy*sizeof(int)); CopyToDevice(dvcSendList_xy,sendList_xy,sendCount_xy*sizeof(int));
dvc_CopyToDevice(dvcSendList_XY,sendList_XY,sendCount_XY*sizeof(int)); CopyToDevice(dvcSendList_XY,sendList_XY,sendCount_XY*sizeof(int));
dvc_CopyToDevice(dvcSendList_xY,sendList_xY,sendCount_xY*sizeof(int)); CopyToDevice(dvcSendList_xY,sendList_xY,sendCount_xY*sizeof(int));
dvc_CopyToDevice(dvcSendList_Xy,sendList_Xy,sendCount_Xy*sizeof(int)); CopyToDevice(dvcSendList_Xy,sendList_Xy,sendCount_Xy*sizeof(int));
dvc_CopyToDevice(dvcSendList_xz,sendList_xz,sendCount_xz*sizeof(int)); CopyToDevice(dvcSendList_xz,sendList_xz,sendCount_xz*sizeof(int));
dvc_CopyToDevice(dvcSendList_XZ,sendList_XZ,sendCount_XZ*sizeof(int)); CopyToDevice(dvcSendList_XZ,sendList_XZ,sendCount_XZ*sizeof(int));
dvc_CopyToDevice(dvcSendList_xZ,sendList_xZ,sendCount_xZ*sizeof(int)); CopyToDevice(dvcSendList_xZ,sendList_xZ,sendCount_xZ*sizeof(int));
dvc_CopyToDevice(dvcSendList_Xz,sendList_Xz,sendCount_Xz*sizeof(int)); CopyToDevice(dvcSendList_Xz,sendList_Xz,sendCount_Xz*sizeof(int));
dvc_CopyToDevice(dvcSendList_yz,sendList_yz,sendCount_yz*sizeof(int)); CopyToDevice(dvcSendList_yz,sendList_yz,sendCount_yz*sizeof(int));
dvc_CopyToDevice(dvcSendList_YZ,sendList_YZ,sendCount_YZ*sizeof(int)); CopyToDevice(dvcSendList_YZ,sendList_YZ,sendCount_YZ*sizeof(int));
dvc_CopyToDevice(dvcSendList_yZ,sendList_yZ,sendCount_yZ*sizeof(int)); CopyToDevice(dvcSendList_yZ,sendList_yZ,sendCount_yZ*sizeof(int));
dvc_CopyToDevice(dvcSendList_Yz,sendList_Yz,sendCount_Yz*sizeof(int)); CopyToDevice(dvcSendList_Yz,sendList_Yz,sendCount_Yz*sizeof(int));
//...................................................................................... //......................................................................................
dvc_CopyToDevice(dvcRecvList_x,recvList_x,recvCount_x*sizeof(int)); CopyToDevice(dvcRecvList_x,recvList_x,recvCount_x*sizeof(int));
dvc_CopyToDevice(dvcRecvList_X,recvList_X,recvCount_X*sizeof(int)); CopyToDevice(dvcRecvList_X,recvList_X,recvCount_X*sizeof(int));
dvc_CopyToDevice(dvcRecvList_y,recvList_y,recvCount_y*sizeof(int)); CopyToDevice(dvcRecvList_y,recvList_y,recvCount_y*sizeof(int));
dvc_CopyToDevice(dvcRecvList_Y,recvList_Y,recvCount_Y*sizeof(int)); CopyToDevice(dvcRecvList_Y,recvList_Y,recvCount_Y*sizeof(int));
dvc_CopyToDevice(dvcRecvList_z,recvList_z,recvCount_z*sizeof(int)); CopyToDevice(dvcRecvList_z,recvList_z,recvCount_z*sizeof(int));
dvc_CopyToDevice(dvcRecvList_Z,recvList_Z,recvCount_Z*sizeof(int)); CopyToDevice(dvcRecvList_Z,recvList_Z,recvCount_Z*sizeof(int));
dvc_CopyToDevice(dvcRecvList_xy,recvList_xy,recvCount_xy*sizeof(int)); CopyToDevice(dvcRecvList_xy,recvList_xy,recvCount_xy*sizeof(int));
dvc_CopyToDevice(dvcRecvList_XY,recvList_XY,recvCount_XY*sizeof(int)); CopyToDevice(dvcRecvList_XY,recvList_XY,recvCount_XY*sizeof(int));
dvc_CopyToDevice(dvcRecvList_xY,recvList_xY,recvCount_xY*sizeof(int)); CopyToDevice(dvcRecvList_xY,recvList_xY,recvCount_xY*sizeof(int));
dvc_CopyToDevice(dvcRecvList_Xy,recvList_Xy,recvCount_Xy*sizeof(int)); CopyToDevice(dvcRecvList_Xy,recvList_Xy,recvCount_Xy*sizeof(int));
dvc_CopyToDevice(dvcRecvList_xz,recvList_xz,recvCount_xz*sizeof(int)); CopyToDevice(dvcRecvList_xz,recvList_xz,recvCount_xz*sizeof(int));
dvc_CopyToDevice(dvcRecvList_XZ,recvList_XZ,recvCount_XZ*sizeof(int)); CopyToDevice(dvcRecvList_XZ,recvList_XZ,recvCount_XZ*sizeof(int));
dvc_CopyToDevice(dvcRecvList_xZ,recvList_xZ,recvCount_xZ*sizeof(int)); CopyToDevice(dvcRecvList_xZ,recvList_xZ,recvCount_xZ*sizeof(int));
dvc_CopyToDevice(dvcRecvList_Xz,recvList_Xz,recvCount_Xz*sizeof(int)); CopyToDevice(dvcRecvList_Xz,recvList_Xz,recvCount_Xz*sizeof(int));
dvc_CopyToDevice(dvcRecvList_yz,recvList_yz,recvCount_yz*sizeof(int)); CopyToDevice(dvcRecvList_yz,recvList_yz,recvCount_yz*sizeof(int));
dvc_CopyToDevice(dvcRecvList_YZ,recvList_YZ,recvCount_YZ*sizeof(int)); CopyToDevice(dvcRecvList_YZ,recvList_YZ,recvCount_YZ*sizeof(int));
dvc_CopyToDevice(dvcRecvList_yZ,recvList_yZ,recvCount_yZ*sizeof(int)); CopyToDevice(dvcRecvList_yZ,recvList_yZ,recvCount_yZ*sizeof(int));
dvc_CopyToDevice(dvcRecvList_Yz,recvList_Yz,recvCount_Yz*sizeof(int)); CopyToDevice(dvcRecvList_Yz,recvList_Yz,recvCount_Yz*sizeof(int));
//...................................................................................... //......................................................................................
// Fill in the phase ID from neighboring processors // Fill in the phase ID from neighboring processors
char *sendID_x, *sendID_y, *sendID_z, *sendID_X, *sendID_Y, *sendID_Z; char *sendID_x, *sendID_y, *sendID_z, *sendID_X, *sendID_Y, *sendID_Z;
@ -941,9 +941,9 @@ int main(int argc, char **argv)
//...........device phase ID................................................. //...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n"); if (rank==0) printf ("Copying phase ID to device \n");
char *ID; char *ID;
dvc_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory AllocateDeviceMemory((void **) &ID, N); // Allocate device memory
// Copy to the device // Copy to the device
dvc_CopyToDevice(ID, id, N); CopyToDevice(ID, id, N);
//........................................................................... //...........................................................................
//........................................................................... //...........................................................................
@ -957,23 +957,23 @@ int main(int argc, char **argv)
double *f_even,*f_odd; double *f_even,*f_odd;
double *A_even,*A_odd,*B_even,*B_odd; double *A_even,*A_odd,*B_even,*B_odd;
//........................................................................... //...........................................................................
dvc_AllocateDeviceMemory((void **) &f_even, 10*dist_mem_size); // Allocate device memory AllocateDeviceMemory((void **) &f_even, 10*dist_mem_size); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &f_odd, 9*dist_mem_size); // Allocate device memory AllocateDeviceMemory((void **) &f_odd, 9*dist_mem_size); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &A_even, 4*dist_mem_size); // Allocate device memory AllocateDeviceMemory((void **) &A_even, 4*dist_mem_size); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &A_odd, 3*dist_mem_size); // Allocate device memory AllocateDeviceMemory((void **) &A_odd, 3*dist_mem_size); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &B_even, 4*dist_mem_size); // Allocate device memory AllocateDeviceMemory((void **) &B_even, 4*dist_mem_size); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &B_odd, 3*dist_mem_size); // Allocate device memory AllocateDeviceMemory((void **) &B_odd, 3*dist_mem_size); // Allocate device memory
//........................................................................... //...........................................................................
double *Phi,*Den; double *Phi,*Den;
// double *Copy; // double *Copy;
double *ColorGrad, *Velocity, *Pressure; double *ColorGrad, *Velocity, *Pressure;
//........................................................................... //...........................................................................
dvc_AllocateDeviceMemory((void **) &Phi, dist_mem_size); AllocateDeviceMemory((void **) &Phi, dist_mem_size);
dvc_AllocateDeviceMemory((void **) &Pressure, dist_mem_size); AllocateDeviceMemory((void **) &Pressure, dist_mem_size);
dvc_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size); AllocateDeviceMemory((void **) &Den, 2*dist_mem_size);
// dvc_AllocateDeviceMemory((void **) &Copy, 2*dist_mem_size); // AllocateDeviceMemory((void **) &Copy, 2*dist_mem_size);
dvc_AllocateDeviceMemory((void **) &Velocity, 3*dist_mem_size); AllocateDeviceMemory((void **) &Velocity, 3*dist_mem_size);
dvc_AllocateDeviceMemory((void **) &ColorGrad, 3*dist_mem_size); AllocateDeviceMemory((void **) &ColorGrad, 3*dist_mem_size);
//........................................................................... //...........................................................................
//........................................................................... //...........................................................................
// Phase indicator (in array form as needed by PMMC algorithm) // Phase indicator (in array form as needed by PMMC algorithm)
@ -1168,16 +1168,16 @@ int main(int argc, char **argv)
} }
} }
// Copy the bubble to the device and initialize // Copy the bubble to the device and initialize
dvc_CopyToDevice(ID, id, N); CopyToDevice(ID, id, N);
//........................................................................... //...........................................................................
//........................................................................... //...........................................................................
dvc_InitD3Q19(ID, f_even, f_odd, Nx, Ny, Nz); InitD3Q19(ID, f_even, f_odd, Nx, Ny, Nz);
//...................................................................... //......................................................................
// dvc_InitDenColorDistance(ID, Copy, Phi, SignDist.data, das, dbs, beta, xIntPos, Nx, Ny, Nz, S); // InitDenColorDistance(ID, Copy, Phi, SignDist.data, das, dbs, beta, xIntPos, Nx, Ny, Nz, S);
dvc_InitDenColorDistance(ID, Den, Phi, SignDist.data, das, dbs, beta, xIntPos, Nx, Ny, Nz); InitDenColorDistance(ID, Den, Phi, SignDist.data, das, dbs, beta, xIntPos, Nx, Ny, Nz);
dvc_InitD3Q7(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); InitD3Q7(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz);
dvc_InitD3Q7(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); InitD3Q7(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz);
//...................................................................... //......................................................................
// Once phase has been initialized, map solid to account for 'smeared' interface // Once phase has been initialized, map solid to account for 'smeared' interface
//...................................................................... //......................................................................
@ -1195,40 +1195,40 @@ int main(int argc, char **argv)
// Read in the restart file to CPU buffers // Read in the restart file to CPU buffers
ReadCheckpoint(LocalRestartFile, cDen, cDistEven, cDistOdd, N); ReadCheckpoint(LocalRestartFile, cDen, cDistEven, cDistOdd, N);
// Copy the restart data to the GPU // Copy the restart data to the GPU
dvc_CopyToDevice(f_even,cDistEven,10*N*sizeof(double)); CopyToDevice(f_even,cDistEven,10*N*sizeof(double));
dvc_CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double)); CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double));
dvc_CopyToDevice(Den,cDen,2*N*sizeof(double)); CopyToDevice(Den,cDen,2*N*sizeof(double));
dvc_Barrier(); Barrier();
MPI_Barrier(MPI_COMM_WORLD); MPI_Barrier(MPI_COMM_WORLD);
} }
//************************************************************************* //*************************************************************************
// Compute the phase indicator field and reset Copy, Den // Compute the phase indicator field and reset Copy, Den
//************************************************************************* //*************************************************************************
// dvc_ComputePhi(ID, Phi, Copy, Den, N); // ComputePhi(ID, Phi, Copy, Den, N);
dvc_ComputePhi(ID, Phi, Den, N); ComputePhi(ID, Phi, Den, N);
//************************************************************************* //*************************************************************************
//................................................................................... //...................................................................................
dvc_PackValues(dvcSendList_x, sendCount_x,sendbuf_x, Phi, N); PackValues(dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
dvc_PackValues(dvcSendList_y, sendCount_y,sendbuf_y, Phi, N); PackValues(dvcSendList_y, sendCount_y,sendbuf_y, Phi, N);
dvc_PackValues(dvcSendList_z, sendCount_z,sendbuf_z, Phi, N); PackValues(dvcSendList_z, sendCount_z,sendbuf_z, Phi, N);
dvc_PackValues(dvcSendList_X, sendCount_X,sendbuf_X, Phi, N); PackValues(dvcSendList_X, sendCount_X,sendbuf_X, Phi, N);
dvc_PackValues(dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N); PackValues(dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N);
dvc_PackValues(dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N); PackValues(dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N);
dvc_PackValues(dvcSendList_xy, sendCount_xy,sendbuf_xy, Phi, N); PackValues(dvcSendList_xy, sendCount_xy,sendbuf_xy, Phi, N);
dvc_PackValues(dvcSendList_xY, sendCount_xY,sendbuf_xY, Phi, N); PackValues(dvcSendList_xY, sendCount_xY,sendbuf_xY, Phi, N);
dvc_PackValues(dvcSendList_Xy, sendCount_Xy,sendbuf_Xy, Phi, N); PackValues(dvcSendList_Xy, sendCount_Xy,sendbuf_Xy, Phi, N);
dvc_PackValues(dvcSendList_XY, sendCount_XY,sendbuf_XY, Phi, N); PackValues(dvcSendList_XY, sendCount_XY,sendbuf_XY, Phi, N);
dvc_PackValues(dvcSendList_xz, sendCount_xz,sendbuf_xz, Phi, N); PackValues(dvcSendList_xz, sendCount_xz,sendbuf_xz, Phi, N);
dvc_PackValues(dvcSendList_xZ, sendCount_xZ,sendbuf_xZ, Phi, N); PackValues(dvcSendList_xZ, sendCount_xZ,sendbuf_xZ, Phi, N);
dvc_PackValues(dvcSendList_Xz, sendCount_Xz,sendbuf_Xz, Phi, N); PackValues(dvcSendList_Xz, sendCount_Xz,sendbuf_Xz, Phi, N);
dvc_PackValues(dvcSendList_XZ, sendCount_XZ,sendbuf_XZ, Phi, N); PackValues(dvcSendList_XZ, sendCount_XZ,sendbuf_XZ, Phi, N);
dvc_PackValues(dvcSendList_yz, sendCount_yz,sendbuf_yz, Phi, N); PackValues(dvcSendList_yz, sendCount_yz,sendbuf_yz, Phi, N);
dvc_PackValues(dvcSendList_yZ, sendCount_yZ,sendbuf_yZ, Phi, N); PackValues(dvcSendList_yZ, sendCount_yZ,sendbuf_yZ, Phi, N);
dvc_PackValues(dvcSendList_Yz, sendCount_Yz,sendbuf_Yz, Phi, N); PackValues(dvcSendList_Yz, sendCount_Yz,sendbuf_Yz, Phi, N);
dvc_PackValues(dvcSendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N); PackValues(dvcSendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N);
dvc_Barrier(); Barrier();
//................................................................................... //...................................................................................
// Send / Recv all the phase indcator field values // Send / Recv all the phase indcator field values
//................................................................................... //...................................................................................
@ -1274,34 +1274,34 @@ int main(int argc, char **argv)
//................................................................................... //...................................................................................
MPI_Waitall(18,req1,stat1); MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2); MPI_Waitall(18,req2,stat2);
dvc_Barrier(); Barrier();
//................................................................................... //...................................................................................
//................................................................................... //...................................................................................
/* dvc_UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N); /* UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
dvc_UnpackValues(faceGrid, packThreads, dvcSendList_y, sendCount_y,sendbuf_y, Phi, N); UnpackValues(faceGrid, packThreads, dvcSendList_y, sendCount_y,sendbuf_y, Phi, N);
dvc_UnpackValues(faceGrid, packThreads, dvcSendList_z, sendCount_z,sendbuf_z, Phi, N); UnpackValues(faceGrid, packThreads, dvcSendList_z, sendCount_z,sendbuf_z, Phi, N);
dvc_UnpackValues(faceGrid, packThreads, dvcSendList_X, sendCount_X,sendbuf_X, Phi, N); UnpackValues(faceGrid, packThreads, dvcSendList_X, sendCount_X,sendbuf_X, Phi, N);
dvc_UnpackValues(faceGrid, packThreads, dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N); UnpackValues(faceGrid, packThreads, dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N);
dvc_UnpackValues(faceGrid, packThreads, dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N); UnpackValues(faceGrid, packThreads, dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N);
*/ */
dvc_UnpackValues(dvcRecvList_x, recvCount_x,recvbuf_x, Phi, N); UnpackValues(dvcRecvList_x, recvCount_x,recvbuf_x, Phi, N);
dvc_UnpackValues(dvcRecvList_y, recvCount_y,recvbuf_y, Phi, N); UnpackValues(dvcRecvList_y, recvCount_y,recvbuf_y, Phi, N);
dvc_UnpackValues(dvcRecvList_z, recvCount_z,recvbuf_z, Phi, N); UnpackValues(dvcRecvList_z, recvCount_z,recvbuf_z, Phi, N);
dvc_UnpackValues(dvcRecvList_X, recvCount_X,recvbuf_X, Phi, N); UnpackValues(dvcRecvList_X, recvCount_X,recvbuf_X, Phi, N);
dvc_UnpackValues(dvcRecvList_Y, recvCount_Y,recvbuf_Y, Phi, N); UnpackValues(dvcRecvList_Y, recvCount_Y,recvbuf_Y, Phi, N);
dvc_UnpackValues(dvcRecvList_Z, recvCount_Z,recvbuf_Z, Phi, N); UnpackValues(dvcRecvList_Z, recvCount_Z,recvbuf_Z, Phi, N);
dvc_UnpackValues(dvcRecvList_xy, recvCount_xy,recvbuf_xy, Phi, N); UnpackValues(dvcRecvList_xy, recvCount_xy,recvbuf_xy, Phi, N);
dvc_UnpackValues(dvcRecvList_xY, recvCount_xY,recvbuf_xY, Phi, N); UnpackValues(dvcRecvList_xY, recvCount_xY,recvbuf_xY, Phi, N);
dvc_UnpackValues(dvcRecvList_Xy, recvCount_Xy,recvbuf_Xy, Phi, N); UnpackValues(dvcRecvList_Xy, recvCount_Xy,recvbuf_Xy, Phi, N);
dvc_UnpackValues(dvcRecvList_XY, recvCount_XY,recvbuf_XY, Phi, N); UnpackValues(dvcRecvList_XY, recvCount_XY,recvbuf_XY, Phi, N);
dvc_UnpackValues(dvcRecvList_xz, recvCount_xz,recvbuf_xz, Phi, N); UnpackValues(dvcRecvList_xz, recvCount_xz,recvbuf_xz, Phi, N);
dvc_UnpackValues(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, Phi, N); UnpackValues(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, Phi, N);
dvc_UnpackValues(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, Phi, N); UnpackValues(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, Phi, N);
dvc_UnpackValues(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, Phi, N); UnpackValues(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, Phi, N);
dvc_UnpackValues(dvcRecvList_yz, recvCount_yz,recvbuf_yz, Phi, N); UnpackValues(dvcRecvList_yz, recvCount_yz,recvbuf_yz, Phi, N);
dvc_UnpackValues(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, Phi, N); UnpackValues(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, Phi, N);
dvc_UnpackValues(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N); UnpackValues(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N);
dvc_UnpackValues(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N); UnpackValues(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
//................................................................................... //...................................................................................
if (rank==0 && pBC){ if (rank==0 && pBC){
@ -1309,29 +1309,29 @@ int main(int argc, char **argv)
printf("Setting outlet pressure = %f \n", dout); printf("Setting outlet pressure = %f \n", dout);
} }
if (pBC && kproc == 0) { if (pBC && kproc == 0) {
dvc_PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz);
dvc_ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
} }
if (pBC && kproc == nprocz-1){ if (pBC && kproc == nprocz-1){
dvc_PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2));
dvc_ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
} }
//........................................................................... //...........................................................................
// Copy the phase indicator field for the earlier timestep // Copy the phase indicator field for the earlier timestep
dvc_Barrier(); Barrier();
dvc_CopyToHost(Phase_tplus.data,Phi,N*sizeof(double)); CopyToHost(Phase_tplus.data,Phi,N*sizeof(double));
//........................................................................... //...........................................................................
//........................................................................... //...........................................................................
// Copy the data for for the analysis timestep // Copy the data for for the analysis timestep
//........................................................................... //...........................................................................
// Copy the phase from the GPU -> CPU // Copy the phase from the GPU -> CPU
//........................................................................... //...........................................................................
dvc_Barrier(); Barrier();
dvc_ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz);
dvc_CopyToHost(Phase.data,Phi,N*sizeof(double)); CopyToHost(Phase.data,Phi,N*sizeof(double));
dvc_CopyToHost(Press.data,Pressure,N*sizeof(double)); CopyToHost(Press.data,Pressure,N*sizeof(double));
MPI_Barrier(MPI_COMM_WORLD); MPI_Barrier(MPI_COMM_WORLD);
//........................................................................... //...........................................................................
@ -1345,71 +1345,71 @@ int main(int argc, char **argv)
//************************************************************************* //*************************************************************************
// Fused Color Gradient and Collision // Fused Color Gradient and Collision
//************************************************************************* //*************************************************************************
dvc_ColorCollideOpt( ID,f_even,f_odd,Phi,ColorGrad, ColorCollideOpt( ID,f_even,f_odd,Phi,ColorGrad,
Velocity,Nx,Ny,Nz,rlxA,rlxB,alpha,beta,Fx,Fy,Fz); Velocity,Nx,Ny,Nz,rlxA,rlxB,alpha,beta,Fx,Fy,Fz);
//************************************************************************* //*************************************************************************
//................................................................................... //...................................................................................
dvc_PackDist(1,dvcSendList_x,0,sendCount_x,sendbuf_x,f_even,N); PackDist(1,dvcSendList_x,0,sendCount_x,sendbuf_x,f_even,N);
dvc_PackDist(4,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,f_even,N); PackDist(4,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,f_even,N);
dvc_PackDist(5,dvcSendList_x,2*sendCount_x,sendCount_x,sendbuf_x,f_even,N); PackDist(5,dvcSendList_x,2*sendCount_x,sendCount_x,sendbuf_x,f_even,N);
dvc_PackDist(6,dvcSendList_x,3*sendCount_x,sendCount_x,sendbuf_x,f_even,N); PackDist(6,dvcSendList_x,3*sendCount_x,sendCount_x,sendbuf_x,f_even,N);
dvc_PackDist(7,dvcSendList_x,4*sendCount_x,sendCount_x,sendbuf_x,f_even,N); PackDist(7,dvcSendList_x,4*sendCount_x,sendCount_x,sendbuf_x,f_even,N);
//...Packing for X face(1,7,9,11,13)................................ //...Packing for X face(1,7,9,11,13)................................
dvc_PackDist(0,dvcSendList_X,0,sendCount_X,sendbuf_X,f_odd,N); PackDist(0,dvcSendList_X,0,sendCount_X,sendbuf_X,f_odd,N);
dvc_PackDist(3,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,f_odd,N); PackDist(3,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,f_odd,N);
dvc_PackDist(4,dvcSendList_X,2*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); PackDist(4,dvcSendList_X,2*sendCount_X,sendCount_X,sendbuf_X,f_odd,N);
dvc_PackDist(5,dvcSendList_X,3*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); PackDist(5,dvcSendList_X,3*sendCount_X,sendCount_X,sendbuf_X,f_odd,N);
dvc_PackDist(6,dvcSendList_X,4*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); PackDist(6,dvcSendList_X,4*sendCount_X,sendCount_X,sendbuf_X,f_odd,N);
//...Packing for y face(4,8,9,16,18)................................. //...Packing for y face(4,8,9,16,18).................................
dvc_PackDist(2,dvcSendList_y,0,sendCount_y,sendbuf_y,f_even,N); PackDist(2,dvcSendList_y,0,sendCount_y,sendbuf_y,f_even,N);
dvc_PackDist(4,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,f_even,N); PackDist(4,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,f_even,N);
dvc_PackDist(4,dvcSendList_y,2*sendCount_y,sendCount_y,sendbuf_y,f_odd,N); PackDist(4,dvcSendList_y,2*sendCount_y,sendCount_y,sendbuf_y,f_odd,N);
dvc_PackDist(8,dvcSendList_y,3*sendCount_y,sendCount_y,sendbuf_y,f_even,N); PackDist(8,dvcSendList_y,3*sendCount_y,sendCount_y,sendbuf_y,f_even,N);
dvc_PackDist(9,dvcSendList_y,4*sendCount_y,sendCount_y,sendbuf_y,f_even,N); PackDist(9,dvcSendList_y,4*sendCount_y,sendCount_y,sendbuf_y,f_even,N);
//...Packing for Y face(3,7,10,15,17)................................. //...Packing for Y face(3,7,10,15,17).................................
dvc_PackDist(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,f_odd,N); PackDist(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,f_odd,N);
dvc_PackDist(3,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); PackDist(3,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N);
dvc_PackDist(5,dvcSendList_Y,2*sendCount_Y,sendCount_Y,sendbuf_Y,f_even,N); PackDist(5,dvcSendList_Y,2*sendCount_Y,sendCount_Y,sendbuf_Y,f_even,N);
dvc_PackDist(7,dvcSendList_Y,3*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); PackDist(7,dvcSendList_Y,3*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N);
dvc_PackDist(8,dvcSendList_Y,4*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); PackDist(8,dvcSendList_Y,4*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N);
//...Packing for z face(6,12,13,16,17)................................ //...Packing for z face(6,12,13,16,17)................................
dvc_PackDist(3,dvcSendList_z,0,sendCount_z,sendbuf_z,f_even,N); PackDist(3,dvcSendList_z,0,sendCount_z,sendbuf_z,f_even,N);
dvc_PackDist(6,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,f_even,N); PackDist(6,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,f_even,N);
dvc_PackDist(6,dvcSendList_z,2*sendCount_z,sendCount_z,sendbuf_z,f_odd,N); PackDist(6,dvcSendList_z,2*sendCount_z,sendCount_z,sendbuf_z,f_odd,N);
dvc_PackDist(8,dvcSendList_z,3*sendCount_z,sendCount_z,sendbuf_z,f_even,N); PackDist(8,dvcSendList_z,3*sendCount_z,sendCount_z,sendbuf_z,f_even,N);
dvc_PackDist(8,dvcSendList_z,4*sendCount_z,sendCount_z,sendbuf_z,f_odd,N); PackDist(8,dvcSendList_z,4*sendCount_z,sendCount_z,sendbuf_z,f_odd,N);
//...Packing for Z face(5,11,14,15,18)................................ //...Packing for Z face(5,11,14,15,18)................................
dvc_PackDist(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,f_odd,N); PackDist(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,f_odd,N);
dvc_PackDist(5,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); PackDist(5,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N);
dvc_PackDist(7,dvcSendList_Z,2*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); PackDist(7,dvcSendList_Z,2*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N);
dvc_PackDist(7,dvcSendList_Z,3*sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); PackDist(7,dvcSendList_Z,3*sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N);
dvc_PackDist(9,dvcSendList_Z,4*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); PackDist(9,dvcSendList_Z,4*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N);
//...Pack the xy edge (8)................................ //...Pack the xy edge (8)................................
dvc_PackDist(4,dvcSendList_xy,0,sendCount_xy,sendbuf_xy,f_even,N); PackDist(4,dvcSendList_xy,0,sendCount_xy,sendbuf_xy,f_even,N);
//...Pack the Xy edge (9)................................ //...Pack the Xy edge (9)................................
dvc_PackDist(4,dvcSendList_Xy,0,sendCount_Xy,sendbuf_Xy,f_odd,N); PackDist(4,dvcSendList_Xy,0,sendCount_Xy,sendbuf_Xy,f_odd,N);
//...Pack the xY edge (10)................................ //...Pack the xY edge (10)................................
dvc_PackDist(5,dvcSendList_xY,0,sendCount_xY,sendbuf_xY,f_even,N); PackDist(5,dvcSendList_xY,0,sendCount_xY,sendbuf_xY,f_even,N);
//...Pack the XY edge (7)................................ //...Pack the XY edge (7)................................
dvc_PackDist(3,dvcSendList_XY,0,sendCount_XY,sendbuf_XY,f_odd,N); PackDist(3,dvcSendList_XY,0,sendCount_XY,sendbuf_XY,f_odd,N);
//...Pack the xz edge (12)................................ //...Pack the xz edge (12)................................
dvc_PackDist(6,dvcSendList_xz,0,sendCount_xz,sendbuf_xz,f_even,N); PackDist(6,dvcSendList_xz,0,sendCount_xz,sendbuf_xz,f_even,N);
//...Pack the xZ edge (14)................................ //...Pack the xZ edge (14)................................
dvc_PackDist(7,dvcSendList_xZ,0,sendCount_xZ,sendbuf_xZ,f_even,N); PackDist(7,dvcSendList_xZ,0,sendCount_xZ,sendbuf_xZ,f_even,N);
//...Pack the Xz edge (13)................................ //...Pack the Xz edge (13)................................
dvc_PackDist(6,dvcSendList_Xz,0,sendCount_Xz,sendbuf_Xz,f_odd,N); PackDist(6,dvcSendList_Xz,0,sendCount_Xz,sendbuf_Xz,f_odd,N);
//...Pack the XZ edge (11)................................ //...Pack the XZ edge (11)................................
dvc_PackDist(5,dvcSendList_XZ,0,sendCount_XZ,sendbuf_XZ,f_odd,N); PackDist(5,dvcSendList_XZ,0,sendCount_XZ,sendbuf_XZ,f_odd,N);
//...Pack the xz edge (12)................................ //...Pack the xz edge (12)................................
//...Pack the yz edge (16)................................ //...Pack the yz edge (16)................................
dvc_PackDist(8,dvcSendList_yz,0,sendCount_yz,sendbuf_yz,f_even,N); PackDist(8,dvcSendList_yz,0,sendCount_yz,sendbuf_yz,f_even,N);
//...Pack the yZ edge (18)................................ //...Pack the yZ edge (18)................................
dvc_PackDist(9,dvcSendList_yZ,0,sendCount_yZ,sendbuf_yZ,f_even,N); PackDist(9,dvcSendList_yZ,0,sendCount_yZ,sendbuf_yZ,f_even,N);
//...Pack the Yz edge (17)................................ //...Pack the Yz edge (17)................................
dvc_PackDist(8,dvcSendList_Yz,0,sendCount_Yz,sendbuf_Yz,f_odd,N); PackDist(8,dvcSendList_Yz,0,sendCount_Yz,sendbuf_Yz,f_odd,N);
//...Pack the YZ edge (15)................................ //...Pack the YZ edge (15)................................
dvc_PackDist(7,dvcSendList_YZ,0,sendCount_YZ,sendbuf_YZ,f_odd,N); PackDist(7,dvcSendList_YZ,0,sendCount_YZ,sendbuf_YZ,f_odd,N);
//................................................................................... //...................................................................................
//................................................................................... //...................................................................................
@ -1455,16 +1455,16 @@ int main(int argc, char **argv)
//************************************************************************* //*************************************************************************
// Carry out the density streaming step for mass transport // Carry out the density streaming step for mass transport
//************************************************************************* //*************************************************************************
// dvc_DensityStreamD3Q7(ID, Den, Copy, Phi, ColorGrad, Velocity, beta, Nx, Ny, Nz, pBC); // DensityStreamD3Q7(ID, Den, Copy, Phi, ColorGrad, Velocity, beta, Nx, Ny, Nz, pBC);
//************************************************************************* //*************************************************************************
dvc_MassColorCollideD3Q7(ID, A_even, A_odd, B_even, B_odd, Den, Phi, MassColorCollideD3Q7(ID, A_even, A_odd, B_even, B_odd, Den, Phi,
ColorGrad, Velocity, beta, N, pBC); ColorGrad, Velocity, beta, N, pBC);
//************************************************************************* //*************************************************************************
// Swap the distributions for momentum transport // Swap the distributions for momentum transport
//************************************************************************* //*************************************************************************
dvc_SwapD3Q19(ID, f_even, f_odd, Nx, Ny, Nz); SwapD3Q19(ID, f_even, f_odd, Nx, Ny, Nz);
//************************************************************************* //*************************************************************************
//................................................................................... //...................................................................................
@ -1476,90 +1476,90 @@ int main(int argc, char **argv)
// Unpack the distributions on the device // Unpack the distributions on the device
//................................................................................... //...................................................................................
//...Map recieve list for the X face: q=2,8,10,12,13 ................................. //...Map recieve list for the X face: q=2,8,10,12,13 .................................
dvc_UnpackDist(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); UnpackDist(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz);
dvc_UnpackDist(3,-1,-1,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); UnpackDist(3,-1,-1,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz);
dvc_UnpackDist(4,-1,1,0,dvcRecvList_X,2*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); UnpackDist(4,-1,1,0,dvcRecvList_X,2*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz);
dvc_UnpackDist(5,-1,0,-1,dvcRecvList_X,3*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); UnpackDist(5,-1,0,-1,dvcRecvList_X,3*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz);
dvc_UnpackDist(6,-1,0,1,dvcRecvList_X,4*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); UnpackDist(6,-1,0,1,dvcRecvList_X,4*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz);
//................................................................................... //...................................................................................
//...Map recieve list for the x face: q=1,7,9,11,13.................................. //...Map recieve list for the x face: q=1,7,9,11,13..................................
dvc_UnpackDist(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); UnpackDist(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz);
dvc_UnpackDist(4,1,1,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); UnpackDist(4,1,1,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz);
dvc_UnpackDist(5,1,-1,0,dvcRecvList_x,2*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); UnpackDist(5,1,-1,0,dvcRecvList_x,2*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz);
dvc_UnpackDist(6,1,0,1,dvcRecvList_x,3*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); UnpackDist(6,1,0,1,dvcRecvList_x,3*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz);
dvc_UnpackDist(7,1,0,-1,dvcRecvList_x,4*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); UnpackDist(7,1,0,-1,dvcRecvList_x,4*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz);
//................................................................................... //...................................................................................
//...Map recieve list for the y face: q=4,8,9,16,18 ................................... //...Map recieve list for the y face: q=4,8,9,16,18 ...................................
dvc_UnpackDist(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); UnpackDist(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz);
dvc_UnpackDist(3,-1,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); UnpackDist(3,-1,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz);
dvc_UnpackDist(5,1,-1,0,dvcRecvList_Y,2*recvCount_Y,recvCount_Y,recvbuf_Y,f_even,Nx,Ny,Nz); UnpackDist(5,1,-1,0,dvcRecvList_Y,2*recvCount_Y,recvCount_Y,recvbuf_Y,f_even,Nx,Ny,Nz);
dvc_UnpackDist(7,0,-1,-1,dvcRecvList_Y,3*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); UnpackDist(7,0,-1,-1,dvcRecvList_Y,3*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz);
dvc_UnpackDist(8,0,-1,1,dvcRecvList_Y,4*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); UnpackDist(8,0,-1,1,dvcRecvList_Y,4*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz);
//................................................................................... //...................................................................................
//...Map recieve list for the Y face: q=3,7,10,15,17 .................................. //...Map recieve list for the Y face: q=3,7,10,15,17 ..................................
dvc_UnpackDist(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); UnpackDist(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz);
dvc_UnpackDist(4,1,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); UnpackDist(4,1,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz);
dvc_UnpackDist(4,-1,1,0,dvcRecvList_y,2*recvCount_y,recvCount_y,recvbuf_y,f_odd,Nx,Ny,Nz); UnpackDist(4,-1,1,0,dvcRecvList_y,2*recvCount_y,recvCount_y,recvbuf_y,f_odd,Nx,Ny,Nz);
dvc_UnpackDist(8,0,1,1,dvcRecvList_y,3*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); UnpackDist(8,0,1,1,dvcRecvList_y,3*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz);
dvc_UnpackDist(9,0,1,-1,dvcRecvList_y,4*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); UnpackDist(9,0,1,-1,dvcRecvList_y,4*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz);
//................................................................................... //...................................................................................
//...Map recieve list for the z face<<<6,12,13,16,17).............................................. //...Map recieve list for the z face<<<6,12,13,16,17)..............................................
dvc_UnpackDist(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); UnpackDist(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz);
dvc_UnpackDist(5,-1,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); UnpackDist(5,-1,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz);
dvc_UnpackDist(7,1,0,-1,dvcRecvList_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz); UnpackDist(7,1,0,-1,dvcRecvList_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz);
dvc_UnpackDist(7,0,-1,-1,dvcRecvList_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); UnpackDist(7,0,-1,-1,dvcRecvList_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz);
dvc_UnpackDist(9,0,1,-1,dvcRecvList_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz); UnpackDist(9,0,1,-1,dvcRecvList_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz);
//...Map recieve list for the Z face<<<5,11,14,15,18).............................................. //...Map recieve list for the Z face<<<5,11,14,15,18)..............................................
dvc_UnpackDist(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); UnpackDist(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz);
dvc_UnpackDist(6,1,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); UnpackDist(6,1,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz);
dvc_UnpackDist(6,-1,0,1,dvcRecvList_z,2*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz); UnpackDist(6,-1,0,1,dvcRecvList_z,2*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz);
dvc_UnpackDist(8,0,1,1,dvcRecvList_z,3*recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); UnpackDist(8,0,1,1,dvcRecvList_z,3*recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz);
dvc_UnpackDist(8,0,-1,1,dvcRecvList_z,4*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz); UnpackDist(8,0,-1,1,dvcRecvList_z,4*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz);
//.................................................................................. //..................................................................................
//...Map recieve list for the xy edge <<<8)................................ //...Map recieve list for the xy edge <<<8)................................
dvc_UnpackDist(3,-1,-1,0,dvcRecvList_XY,0,recvCount_XY,recvbuf_XY,f_odd,Nx,Ny,Nz); UnpackDist(3,-1,-1,0,dvcRecvList_XY,0,recvCount_XY,recvbuf_XY,f_odd,Nx,Ny,Nz);
//...Map recieve list for the Xy edge <<<9)................................ //...Map recieve list for the Xy edge <<<9)................................
dvc_UnpackDist(5,1,-1,0,dvcRecvList_xY,0,recvCount_xY,recvbuf_xY,f_even,Nx,Ny,Nz); UnpackDist(5,1,-1,0,dvcRecvList_xY,0,recvCount_xY,recvbuf_xY,f_even,Nx,Ny,Nz);
//...Map recieve list for the xY edge <<<10)................................ //...Map recieve list for the xY edge <<<10)................................
dvc_UnpackDist(4,-1,1,0,dvcRecvList_Xy,0,recvCount_Xy,recvbuf_Xy,f_odd,Nx,Ny,Nz); UnpackDist(4,-1,1,0,dvcRecvList_Xy,0,recvCount_Xy,recvbuf_Xy,f_odd,Nx,Ny,Nz);
//...Map recieve list for the XY edge <<<7)................................ //...Map recieve list for the XY edge <<<7)................................
dvc_UnpackDist(4,1,1,0,dvcRecvList_xy,0,recvCount_xy,recvbuf_xy,f_even,Nx,Ny,Nz); UnpackDist(4,1,1,0,dvcRecvList_xy,0,recvCount_xy,recvbuf_xy,f_even,Nx,Ny,Nz);
//...Map recieve list for the xz edge <<<12)................................ //...Map recieve list for the xz edge <<<12)................................
dvc_UnpackDist(5,-1,0,-1,dvcRecvList_XZ,0,recvCount_XZ,recvbuf_XZ,f_odd,Nx,Ny,Nz); UnpackDist(5,-1,0,-1,dvcRecvList_XZ,0,recvCount_XZ,recvbuf_XZ,f_odd,Nx,Ny,Nz);
//...Map recieve list for the xZ edge <<<14)................................ //...Map recieve list for the xZ edge <<<14)................................
dvc_UnpackDist(6,-1,0,1,dvcRecvList_Xz,0,recvCount_Xz,recvbuf_Xz,f_odd,Nx,Ny,Nz); UnpackDist(6,-1,0,1,dvcRecvList_Xz,0,recvCount_Xz,recvbuf_Xz,f_odd,Nx,Ny,Nz);
//...Map recieve list for the Xz edge <<<13)................................ //...Map recieve list for the Xz edge <<<13)................................
dvc_UnpackDist(7,1,0,-1,dvcRecvList_xZ,0,recvCount_xZ,recvbuf_xZ,f_even,Nx,Ny,Nz); UnpackDist(7,1,0,-1,dvcRecvList_xZ,0,recvCount_xZ,recvbuf_xZ,f_even,Nx,Ny,Nz);
//...Map recieve list for the XZ edge <<<11)................................ //...Map recieve list for the XZ edge <<<11)................................
dvc_UnpackDist(6,1,0,1,dvcRecvList_xz,0,recvCount_xz,recvbuf_xz,f_even,Nx,Ny,Nz); UnpackDist(6,1,0,1,dvcRecvList_xz,0,recvCount_xz,recvbuf_xz,f_even,Nx,Ny,Nz);
//...Map recieve list for the yz edge <<<16)................................ //...Map recieve list for the yz edge <<<16)................................
dvc_UnpackDist(7,0,-1,-1,dvcRecvList_YZ,0,recvCount_YZ,recvbuf_YZ,f_odd,Nx,Ny,Nz); UnpackDist(7,0,-1,-1,dvcRecvList_YZ,0,recvCount_YZ,recvbuf_YZ,f_odd,Nx,Ny,Nz);
//...Map recieve list for the yZ edge <<<18)................................ //...Map recieve list for the yZ edge <<<18)................................
dvc_UnpackDist(8,0,-1,1,dvcRecvList_Yz,0,recvCount_Yz,recvbuf_Yz,f_odd,Nx,Ny,Nz); UnpackDist(8,0,-1,1,dvcRecvList_Yz,0,recvCount_Yz,recvbuf_Yz,f_odd,Nx,Ny,Nz);
//...Map recieve list for the Yz edge <<<17)................................ //...Map recieve list for the Yz edge <<<17)................................
dvc_UnpackDist(9,0,1,-1,dvcRecvList_yZ,0,recvCount_yZ,recvbuf_yZ,f_even,Nx,Ny,Nz); UnpackDist(9,0,1,-1,dvcRecvList_yZ,0,recvCount_yZ,recvbuf_yZ,f_even,Nx,Ny,Nz);
//...Map recieve list for the YZ edge <<<15)................................ //...Map recieve list for the YZ edge <<<15)................................
dvc_UnpackDist(8,0,1,1,dvcRecvList_yz,0,recvCount_yz,recvbuf_yz,f_even,Nx,Ny,Nz); UnpackDist(8,0,1,1,dvcRecvList_yz,0,recvCount_yz,recvbuf_yz,f_even,Nx,Ny,Nz);
//................................................................................... //...................................................................................
//................................................................................... //...................................................................................
dvc_PackDist(1,dvcSendList_x,0,sendCount_x,sendbuf_x,A_even,N); PackDist(1,dvcSendList_x,0,sendCount_x,sendbuf_x,A_even,N);
dvc_PackDist(1,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,B_even,N); PackDist(1,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,B_even,N);
//...Packing for X face(1,7,9,11,13)................................ //...Packing for X face(1,7,9,11,13)................................
dvc_PackDist(0,dvcSendList_X,0,sendCount_X,sendbuf_X,A_odd,N); PackDist(0,dvcSendList_X,0,sendCount_X,sendbuf_X,A_odd,N);
dvc_PackDist(0,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,B_odd,N); PackDist(0,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,B_odd,N);
//...Packing for y face(4,8,9,16,18)................................. //...Packing for y face(4,8,9,16,18).................................
dvc_PackDist(2,dvcSendList_y,0,sendCount_y,sendbuf_y,A_even,N); PackDist(2,dvcSendList_y,0,sendCount_y,sendbuf_y,A_even,N);
dvc_PackDist(2,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,B_even,N); PackDist(2,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,B_even,N);
//...Packing for Y face(3,7,10,15,17)................................. //...Packing for Y face(3,7,10,15,17).................................
dvc_PackDist(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,A_odd,N); PackDist(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,A_odd,N);
dvc_PackDist(1,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,B_odd,N); PackDist(1,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,B_odd,N);
//...Packing for z face(6,12,13,16,17)................................ //...Packing for z face(6,12,13,16,17)................................
dvc_PackDist(3,dvcSendList_z,0,sendCount_z,sendbuf_z,A_even,N); PackDist(3,dvcSendList_z,0,sendCount_z,sendbuf_z,A_even,N);
dvc_PackDist(3,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,B_even,N); PackDist(3,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,B_even,N);
//...Packing for Z face(5,11,14,15,18)................................ //...Packing for Z face(5,11,14,15,18)................................
dvc_PackDist(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,A_odd,N); PackDist(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,A_odd,N);
dvc_PackDist(2,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,B_odd,N); PackDist(2,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,B_odd,N);
//................................................................................... //...................................................................................
//................................................................................... //...................................................................................
@ -1578,8 +1578,8 @@ int main(int argc, char **argv)
MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]); MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
//................................................................................... //...................................................................................
dvc_SwapD3Q7(ID, A_even, A_odd, Nx, Ny, Nz); SwapD3Q7(ID, A_even, A_odd, Nx, Ny, Nz);
dvc_SwapD3Q7(ID, B_even, B_odd, Nx, Ny, Nz); SwapD3Q7(ID, B_even, B_odd, Nx, Ny, Nz);
//................................................................................... //...................................................................................
// Wait for completion of D3Q19 communication // Wait for completion of D3Q19 communication
@ -1589,59 +1589,59 @@ int main(int argc, char **argv)
// Unpack the distributions on the device // Unpack the distributions on the device
//................................................................................... //...................................................................................
//...Map recieve list for the X face: q=2,8,10,12,13 ................................. //...Map recieve list for the X face: q=2,8,10,12,13 .................................
dvc_UnpackDist(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,A_odd,Nx,Ny,Nz); UnpackDist(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,A_odd,Nx,Ny,Nz);
dvc_UnpackDist(0,-1,0,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,B_odd,Nx,Ny,Nz); UnpackDist(0,-1,0,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,B_odd,Nx,Ny,Nz);
//................................................................................... //...................................................................................
//...Map recieve list for the x face: q=1,7,9,11,13.................................. //...Map recieve list for the x face: q=1,7,9,11,13..................................
dvc_UnpackDist(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,A_even,Nx,Ny,Nz); UnpackDist(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,A_even,Nx,Ny,Nz);
dvc_UnpackDist(1,1,0,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,B_even,Nx,Ny,Nz); UnpackDist(1,1,0,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,B_even,Nx,Ny,Nz);
//................................................................................... //...................................................................................
//...Map recieve list for the y face: q=4,8,9,16,18 ................................... //...Map recieve list for the y face: q=4,8,9,16,18 ...................................
dvc_UnpackDist(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,A_odd,Nx,Ny,Nz); UnpackDist(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,A_odd,Nx,Ny,Nz);
dvc_UnpackDist(1,0,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,B_odd,Nx,Ny,Nz); UnpackDist(1,0,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,B_odd,Nx,Ny,Nz);
//................................................................................... //...................................................................................
//...Map recieve list for the Y face: q=3,7,10,15,17 .................................. //...Map recieve list for the Y face: q=3,7,10,15,17 ..................................
dvc_UnpackDist(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,A_even,Nx,Ny,Nz); UnpackDist(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,A_even,Nx,Ny,Nz);
dvc_UnpackDist(2,0,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,B_even,Nx,Ny,Nz); UnpackDist(2,0,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,B_even,Nx,Ny,Nz);
//................................................................................... //...................................................................................
//...Map recieve list for the z face<<<6,12,13,16,17).............................................. //...Map recieve list for the z face<<<6,12,13,16,17)..............................................
dvc_UnpackDist(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,A_odd,Nx,Ny,Nz); UnpackDist(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,A_odd,Nx,Ny,Nz);
dvc_UnpackDist(2,0,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,B_odd,Nx,Ny,Nz); UnpackDist(2,0,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,B_odd,Nx,Ny,Nz);
//...Map recieve list for the Z face<<<5,11,14,15,18).............................................. //...Map recieve list for the Z face<<<5,11,14,15,18)..............................................
dvc_UnpackDist(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,A_even,Nx,Ny,Nz); UnpackDist(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,A_even,Nx,Ny,Nz);
dvc_UnpackDist(3,0,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,B_even,Nx,Ny,Nz); UnpackDist(3,0,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,B_even,Nx,Ny,Nz);
//.................................................................................. //..................................................................................
//.................................................................................. //..................................................................................
dvc_ComputeDensityD3Q7(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); ComputeDensityD3Q7(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz);
dvc_ComputeDensityD3Q7(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); ComputeDensityD3Q7(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz);
//************************************************************************* //*************************************************************************
// Compute the phase indicator field // Compute the phase indicator field
//************************************************************************* //*************************************************************************
// dvc_ComputePhi(ID, Phi, Copy, Den, N); // ComputePhi(ID, Phi, Copy, Den, N);
dvc_ComputePhi(ID, Phi, Den, N); ComputePhi(ID, Phi, Den, N);
//************************************************************************* //*************************************************************************
//................................................................................... //...................................................................................
dvc_PackValues(dvcSendList_x, sendCount_x,sendbuf_x, Phi, N); PackValues(dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
dvc_PackValues(dvcSendList_y, sendCount_y,sendbuf_y, Phi, N); PackValues(dvcSendList_y, sendCount_y,sendbuf_y, Phi, N);
dvc_PackValues(dvcSendList_z, sendCount_z,sendbuf_z, Phi, N); PackValues(dvcSendList_z, sendCount_z,sendbuf_z, Phi, N);
dvc_PackValues(dvcSendList_X, sendCount_X,sendbuf_X, Phi, N); PackValues(dvcSendList_X, sendCount_X,sendbuf_X, Phi, N);
dvc_PackValues(dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N); PackValues(dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N);
dvc_PackValues(dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N); PackValues(dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N);
dvc_PackValues(dvcSendList_xy, sendCount_xy,sendbuf_xy, Phi, N); PackValues(dvcSendList_xy, sendCount_xy,sendbuf_xy, Phi, N);
dvc_PackValues(dvcSendList_xY, sendCount_xY,sendbuf_xY, Phi, N); PackValues(dvcSendList_xY, sendCount_xY,sendbuf_xY, Phi, N);
dvc_PackValues(dvcSendList_Xy, sendCount_Xy,sendbuf_Xy, Phi, N); PackValues(dvcSendList_Xy, sendCount_Xy,sendbuf_Xy, Phi, N);
dvc_PackValues(dvcSendList_XY, sendCount_XY,sendbuf_XY, Phi, N); PackValues(dvcSendList_XY, sendCount_XY,sendbuf_XY, Phi, N);
dvc_PackValues(dvcSendList_xz, sendCount_xz,sendbuf_xz, Phi, N); PackValues(dvcSendList_xz, sendCount_xz,sendbuf_xz, Phi, N);
dvc_PackValues(dvcSendList_xZ, sendCount_xZ,sendbuf_xZ, Phi, N); PackValues(dvcSendList_xZ, sendCount_xZ,sendbuf_xZ, Phi, N);
dvc_PackValues(dvcSendList_Xz, sendCount_Xz,sendbuf_Xz, Phi, N); PackValues(dvcSendList_Xz, sendCount_Xz,sendbuf_Xz, Phi, N);
dvc_PackValues(dvcSendList_XZ, sendCount_XZ,sendbuf_XZ, Phi, N); PackValues(dvcSendList_XZ, sendCount_XZ,sendbuf_XZ, Phi, N);
dvc_PackValues(dvcSendList_yz, sendCount_yz,sendbuf_yz, Phi, N); PackValues(dvcSendList_yz, sendCount_yz,sendbuf_yz, Phi, N);
dvc_PackValues(dvcSendList_yZ, sendCount_yZ,sendbuf_yZ, Phi, N); PackValues(dvcSendList_yZ, sendCount_yZ,sendbuf_yZ, Phi, N);
dvc_PackValues(dvcSendList_Yz, sendCount_Yz,sendbuf_Yz, Phi, N); PackValues(dvcSendList_Yz, sendCount_Yz,sendbuf_Yz, Phi, N);
dvc_PackValues(dvcSendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N); PackValues(dvcSendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N);
//................................................................................... //...................................................................................
// Send / Recv all the phase indcator field values // Send / Recv all the phase indcator field values
//................................................................................... //...................................................................................
@ -1687,44 +1687,44 @@ int main(int argc, char **argv)
//................................................................................... //...................................................................................
MPI_Waitall(18,req1,stat1); MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2); MPI_Waitall(18,req2,stat2);
dvc_Barrier(); Barrier();
//................................................................................... //...................................................................................
//................................................................................... //...................................................................................
/* dvc_UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N); /* UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
dvc_UnpackValues(faceGrid, packThreads, dvcSendList_y, sendCount_y,sendbuf_y, Phi, N); UnpackValues(faceGrid, packThreads, dvcSendList_y, sendCount_y,sendbuf_y, Phi, N);
dvc_UnpackValues(faceGrid, packThreads, dvcSendList_z, sendCount_z,sendbuf_z, Phi, N); UnpackValues(faceGrid, packThreads, dvcSendList_z, sendCount_z,sendbuf_z, Phi, N);
dvc_UnpackValues(faceGrid, packThreads, dvcSendList_X, sendCount_X,sendbuf_X, Phi, N); UnpackValues(faceGrid, packThreads, dvcSendList_X, sendCount_X,sendbuf_X, Phi, N);
dvc_UnpackValues(faceGrid, packThreads, dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N); UnpackValues(faceGrid, packThreads, dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N);
dvc_UnpackValues(faceGrid, packThreads, dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N); UnpackValues(faceGrid, packThreads, dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N);
*/ */
dvc_UnpackValues(dvcRecvList_x, recvCount_x,recvbuf_x, Phi, N); UnpackValues(dvcRecvList_x, recvCount_x,recvbuf_x, Phi, N);
dvc_UnpackValues(dvcRecvList_y, recvCount_y,recvbuf_y, Phi, N); UnpackValues(dvcRecvList_y, recvCount_y,recvbuf_y, Phi, N);
dvc_UnpackValues(dvcRecvList_z, recvCount_z,recvbuf_z, Phi, N); UnpackValues(dvcRecvList_z, recvCount_z,recvbuf_z, Phi, N);
dvc_UnpackValues(dvcRecvList_X, recvCount_X,recvbuf_X, Phi, N); UnpackValues(dvcRecvList_X, recvCount_X,recvbuf_X, Phi, N);
dvc_UnpackValues(dvcRecvList_Y, recvCount_Y,recvbuf_Y, Phi, N); UnpackValues(dvcRecvList_Y, recvCount_Y,recvbuf_Y, Phi, N);
dvc_UnpackValues(dvcRecvList_Z, recvCount_Z,recvbuf_Z, Phi, N); UnpackValues(dvcRecvList_Z, recvCount_Z,recvbuf_Z, Phi, N);
dvc_UnpackValues(dvcRecvList_xy, recvCount_xy,recvbuf_xy, Phi, N); UnpackValues(dvcRecvList_xy, recvCount_xy,recvbuf_xy, Phi, N);
dvc_UnpackValues(dvcRecvList_xY, recvCount_xY,recvbuf_xY, Phi, N); UnpackValues(dvcRecvList_xY, recvCount_xY,recvbuf_xY, Phi, N);
dvc_UnpackValues(dvcRecvList_Xy, recvCount_Xy,recvbuf_Xy, Phi, N); UnpackValues(dvcRecvList_Xy, recvCount_Xy,recvbuf_Xy, Phi, N);
dvc_UnpackValues(dvcRecvList_XY, recvCount_XY,recvbuf_XY, Phi, N); UnpackValues(dvcRecvList_XY, recvCount_XY,recvbuf_XY, Phi, N);
dvc_UnpackValues(dvcRecvList_xz, recvCount_xz,recvbuf_xz, Phi, N); UnpackValues(dvcRecvList_xz, recvCount_xz,recvbuf_xz, Phi, N);
dvc_UnpackValues(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, Phi, N); UnpackValues(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, Phi, N);
dvc_UnpackValues(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, Phi, N); UnpackValues(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, Phi, N);
dvc_UnpackValues(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, Phi, N); UnpackValues(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, Phi, N);
dvc_UnpackValues(dvcRecvList_yz, recvCount_yz,recvbuf_yz, Phi, N); UnpackValues(dvcRecvList_yz, recvCount_yz,recvbuf_yz, Phi, N);
dvc_UnpackValues(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, Phi, N); UnpackValues(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, Phi, N);
dvc_UnpackValues(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N); UnpackValues(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N);
dvc_UnpackValues(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N); UnpackValues(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
//................................................................................... //...................................................................................
if (pBC && kproc == 0) { if (pBC && kproc == 0) {
dvc_PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz);
dvc_ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
} }
if (pBC && kproc == nprocz-1){ if (pBC && kproc == nprocz-1){
dvc_PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2));
dvc_ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
} }
//................................................................................... //...................................................................................
@ -1736,9 +1736,9 @@ int main(int argc, char **argv)
if (timestep%RESTART_INTERVAL == 0){ if (timestep%RESTART_INTERVAL == 0){
// Copy the data to the CPU // Copy the data to the CPU
dvc_CopyToHost(cDistEven,f_even,10*N*sizeof(double)); CopyToHost(cDistEven,f_even,10*N*sizeof(double));
dvc_CopyToHost(cDistOdd,f_odd,9*N*sizeof(double)); CopyToHost(cDistOdd,f_odd,9*N*sizeof(double));
dvc_CopyToHost(cDen,Den,2*N*sizeof(double)); CopyToHost(cDen,Den,2*N*sizeof(double));
// Read in the restart file to CPU buffers // Read in the restart file to CPU buffers
WriteCheckpoint(LocalRestartFile, cDen, cDistEven, cDistOdd, N); WriteCheckpoint(LocalRestartFile, cDen, cDistEven, cDistOdd, N);
} }
@ -1746,12 +1746,12 @@ int main(int argc, char **argv)
// End the bubble loop // End the bubble loop
//........................................................................... //...........................................................................
// Copy the phase indicator field for the later timestep // Copy the phase indicator field for the later timestep
dvc_Barrier(); Barrier();
dvc_ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz);
dvc_CopyToHost(Phase_tminus.data,Phi,N*sizeof(double)); CopyToHost(Phase_tminus.data,Phi,N*sizeof(double));
dvc_CopyToHost(Phase_tplus.data,Phi,N*sizeof(double)); CopyToHost(Phase_tplus.data,Phi,N*sizeof(double));
dvc_CopyToHost(Phase.data,Phi,N*sizeof(double)); CopyToHost(Phase.data,Phi,N*sizeof(double));
dvc_CopyToHost(Press.data,Pressure,N*sizeof(double)); CopyToHost(Press.data,Pressure,N*sizeof(double));
//........................................................................... //...........................................................................
// Calculate the time derivative of the phase indicator field // Calculate the time derivative of the phase indicator field
@ -2081,7 +2081,7 @@ int main(int argc, char **argv)
} }
//************************************************************************/ //************************************************************************/
dvc_Barrier(); Barrier();
MPI_Barrier(MPI_COMM_WORLD); MPI_Barrier(MPI_COMM_WORLD);
stoptime = MPI_Wtime(); stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n"); if (rank==0) printf("-------------------------------------------------------------------\n");
@ -2102,7 +2102,7 @@ int main(int argc, char **argv)
//************************************************************************/ //************************************************************************/
sprintf(LocalRankFilename,"%s%s","Phase.",LocalRankString); sprintf(LocalRankFilename,"%s%s","Phase.",LocalRankString);
// printf("Local File Name = %s \n",LocalRankFilename); // printf("Local File Name = %s \n",LocalRankFilename);
// dvc_CopyToHost(Phase.data,Phi,N*sizeof(double)); // CopyToHost(Phase.data,Phi,N*sizeof(double));
FILE *PHASE; FILE *PHASE;
PHASE = fopen(LocalRankFilename,"wb"); PHASE = fopen(LocalRankFilename,"wb");
@ -2112,7 +2112,7 @@ int main(int argc, char **argv)
/* double *DensityValues; /* double *DensityValues;
DensityValues = new double [2*N]; DensityValues = new double [2*N];
dvc_CopyToHost(DensityValues,Copy,2*N*sizeof(double)); CopyToHost(DensityValues,Copy,2*N*sizeof(double));
FILE *PHASE; FILE *PHASE;
PHASE = fopen(LocalRankFilename,"wb"); PHASE = fopen(LocalRankFilename,"wb");
fwrite(DensityValues,8,2*N,PHASE); fwrite(DensityValues,8,2*N,PHASE);