OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

resolve merge conflict

Browse Source
This commit is contained in:
Rex Zhe Li 2020-08-18 18:02:28 -04:00
commit 2180f7b2bf
25 changed files with 26752 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

574
common/ScaLBL.cpp
View File

@ -80,43 +80,43 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr <Domain> Dm){
BoundaryCondition = Dm->BoundaryCondition;
//......................................................................................
ScaLBL_AllocateZeroCopy((void **) &sendbuf_x, 5*sendCount_x*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_X, 5*sendCount_X*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_y, 5*sendCount_y*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Y, 5*sendCount_Y*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_z, 5*sendCount_z*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Z, 5*sendCount_Z*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xy, sendCount_xy*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xY, sendCount_xY*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xy, sendCount_Xy*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_XY, sendCount_XY*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xz, sendCount_xz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xZ, sendCount_xZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xz, sendCount_Xz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_XZ, sendCount_XZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_yz, sendCount_yz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_yZ, sendCount_yZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Yz, sendCount_Yz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_YZ, sendCount_YZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_x, 2*5*sendCount_x*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_X, 2*5*sendCount_X*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_y, 2*5*sendCount_y*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Y, 2*5*sendCount_Y*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_z, 2*5*sendCount_z*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Z, 2*5*sendCount_Z*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xy, 2*sendCount_xy*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xY, 2*sendCount_xY*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xy, 2*sendCount_Xy*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_XY, 2*sendCount_XY*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xz, 2*sendCount_xz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xZ, 2*sendCount_xZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xz, 2*sendCount_Xz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_XZ, 2*sendCount_XZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_yz, 2*sendCount_yz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_yZ, 2*sendCount_yZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Yz, 2*sendCount_Yz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_YZ, 2*sendCount_YZ*sizeof(double)); // Allocate device memory
//......................................................................................
ScaLBL_AllocateZeroCopy((void **) &recvbuf_x, 5*recvCount_x*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_X, 5*recvCount_X*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_y, 5*recvCount_y*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Y, 5*recvCount_Y*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_z, 5*recvCount_z*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Z, 5*recvCount_Z*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xy, recvCount_xy*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xY, recvCount_xY*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xy, recvCount_Xy*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_XY, recvCount_XY*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xz, recvCount_xz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xZ, recvCount_xZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xz, recvCount_Xz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_XZ, recvCount_XZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_yz, recvCount_yz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_yZ, recvCount_yZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Yz, recvCount_Yz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_YZ, recvCount_YZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_x, 2*5*recvCount_x*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_X, 2*5*recvCount_X*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_y, 2*5*recvCount_y*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Y, 2*5*recvCount_Y*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_z, 2*5*recvCount_z*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Z, 2*5*recvCount_Z*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xy, 2*recvCount_xy*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xY, 2*recvCount_xY*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xy, 2*recvCount_Xy*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_XY, 2*recvCount_XY*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xz, 2*recvCount_xz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xZ, 2*recvCount_xZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xz, 2*recvCount_Xz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_XZ, 2*recvCount_XZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_yz, 2*recvCount_yz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_yZ, 2*recvCount_yZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Yz, 2*recvCount_Yz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_YZ, 2*recvCount_YZ*sizeof(double)); // Allocate device memory
//......................................................................................
ScaLBL_AllocateZeroCopy((void **) &dvcSendList_x, sendCount_x*sizeof(int)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &dvcSendList_X, sendCount_X*sizeof(int)); // Allocate device memory
@ -1094,6 +1094,368 @@ void ScaLBL_Communicator::RecvD3Q19AA(double *dist){
}
void ScaLBL_Communicator::BiSendD3Q19AA(double *Aq, double *Bq){
// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
if (Lock==true){
ERROR("ScaLBL Error (SendD3Q19): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?");
}
else{
Lock=true;
}
// assign tag of 19 to D3Q19 communication
sendtag = recvtag = 38;
ScaLBL_DeviceBarrier();
// Pack the distributions
//...Packing for x face(2,8,10,12,14)................................
ScaLBL_D3Q19_Pack(2 ,dvcSendList_x,0*sendCount_x,sendCount_x,sendbuf_x,Aq,N);
ScaLBL_D3Q19_Pack(8 ,dvcSendList_x,1*sendCount_x,sendCount_x,sendbuf_x,Aq,N);
ScaLBL_D3Q19_Pack(10,dvcSendList_x,2*sendCount_x,sendCount_x,sendbuf_x,Aq,N);
ScaLBL_D3Q19_Pack(12,dvcSendList_x,3*sendCount_x,sendCount_x,sendbuf_x,Aq,N);
ScaLBL_D3Q19_Pack(14,dvcSendList_x,4*sendCount_x,sendCount_x,sendbuf_x,Aq,N);
ScaLBL_D3Q19_Pack(2 ,dvcSendList_x,5*sendCount_x,sendCount_x,sendbuf_x,Bq,N);
ScaLBL_D3Q19_Pack(8 ,dvcSendList_x,6*sendCount_x,sendCount_x,sendbuf_x,Bq,N);
ScaLBL_D3Q19_Pack(10,dvcSendList_x,7*sendCount_x,sendCount_x,sendbuf_x,Bq,N);
ScaLBL_D3Q19_Pack(12,dvcSendList_x,8*sendCount_x,sendCount_x,sendbuf_x,Bq,N);
ScaLBL_D3Q19_Pack(14,dvcSendList_x,9*sendCount_x,sendCount_x,sendbuf_x,Bq,N);
req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, 10*sendCount_x,rank_x,sendtag);
req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, 10*recvCount_X,rank_X,recvtag);
//...Packing for X face(1,7,9,11,13)................................
ScaLBL_D3Q19_Pack(1 ,dvcSendList_X,0*sendCount_X,sendCount_X,sendbuf_X,Aq,N);
ScaLBL_D3Q19_Pack(7 ,dvcSendList_X,1*sendCount_X,sendCount_X,sendbuf_X,Aq,N);
ScaLBL_D3Q19_Pack(9 ,dvcSendList_X,2*sendCount_X,sendCount_X,sendbuf_X,Aq,N);
ScaLBL_D3Q19_Pack(11,dvcSendList_X,3*sendCount_X,sendCount_X,sendbuf_X,Aq,N);
ScaLBL_D3Q19_Pack(13,dvcSendList_X,4*sendCount_X,sendCount_X,sendbuf_X,Aq,N);
ScaLBL_D3Q19_Pack(1 ,dvcSendList_X,5*sendCount_X,sendCount_X,sendbuf_X,Bq,N);
ScaLBL_D3Q19_Pack(7 ,dvcSendList_X,6*sendCount_X,sendCount_X,sendbuf_X,Bq,N);
ScaLBL_D3Q19_Pack(9 ,dvcSendList_X,7*sendCount_X,sendCount_X,sendbuf_X,Bq,N);
ScaLBL_D3Q19_Pack(11,dvcSendList_X,8*sendCount_X,sendCount_X,sendbuf_X,Bq,N);
ScaLBL_D3Q19_Pack(13,dvcSendList_X,9*sendCount_X,sendCount_X,sendbuf_X,Bq,N);
req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, 10*sendCount_X,rank_X,sendtag);
req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, 10*recvCount_x,rank_x,recvtag);
//...Packing for y face(4,8,9,16,18).................................
ScaLBL_D3Q19_Pack(4 ,dvcSendList_y,0*sendCount_y,sendCount_y,sendbuf_y,Aq,N);
ScaLBL_D3Q19_Pack(8 ,dvcSendList_y,1*sendCount_y,sendCount_y,sendbuf_y,Aq,N);
ScaLBL_D3Q19_Pack(9 ,dvcSendList_y,2*sendCount_y,sendCount_y,sendbuf_y,Aq,N);
ScaLBL_D3Q19_Pack(16,dvcSendList_y,3*sendCount_y,sendCount_y,sendbuf_y,Aq,N);
ScaLBL_D3Q19_Pack(18,dvcSendList_y,4*sendCount_y,sendCount_y,sendbuf_y,Aq,N);
ScaLBL_D3Q19_Pack(4 ,dvcSendList_y,5*sendCount_y,sendCount_y,sendbuf_y,Bq,N);
ScaLBL_D3Q19_Pack(8 ,dvcSendList_y,6*sendCount_y,sendCount_y,sendbuf_y,Bq,N);
ScaLBL_D3Q19_Pack(9 ,dvcSendList_y,7*sendCount_y,sendCount_y,sendbuf_y,Bq,N);
ScaLBL_D3Q19_Pack(16,dvcSendList_y,8*sendCount_y,sendCount_y,sendbuf_y,Bq,N);
ScaLBL_D3Q19_Pack(18,dvcSendList_y,9*sendCount_y,sendCount_y,sendbuf_y,Bq,N);
req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, 10*sendCount_y,rank_y,sendtag);
req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, 10*recvCount_Y,rank_Y,recvtag);
//...Packing for Y face(3,7,10,15,17).................................
ScaLBL_D3Q19_Pack(3 ,dvcSendList_Y,0*sendCount_Y,sendCount_Y,sendbuf_Y,Aq,N);
ScaLBL_D3Q19_Pack(7 ,dvcSendList_Y,1*sendCount_Y,sendCount_Y,sendbuf_Y,Aq,N);
ScaLBL_D3Q19_Pack(10,dvcSendList_Y,2*sendCount_Y,sendCount_Y,sendbuf_Y,Aq,N);
ScaLBL_D3Q19_Pack(15,dvcSendList_Y,3*sendCount_Y,sendCount_Y,sendbuf_Y,Aq,N);
ScaLBL_D3Q19_Pack(17,dvcSendList_Y,4*sendCount_Y,sendCount_Y,sendbuf_Y,Aq,N);
ScaLBL_D3Q19_Pack(3 ,dvcSendList_Y,5*sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N);
ScaLBL_D3Q19_Pack(7 ,dvcSendList_Y,6*sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N);
ScaLBL_D3Q19_Pack(10,dvcSendList_Y,7*sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N);
ScaLBL_D3Q19_Pack(15,dvcSendList_Y,8*sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N);
ScaLBL_D3Q19_Pack(17,dvcSendList_Y,9*sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N);
req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, 10*sendCount_Y,rank_Y,sendtag);
req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, 10*recvCount_y,rank_y,recvtag);
//...Packing for z face(6,12,13,16,17)................................
ScaLBL_D3Q19_Pack(6 ,dvcSendList_z,0*sendCount_z,sendCount_z,sendbuf_z,Aq,N);
ScaLBL_D3Q19_Pack(12,dvcSendList_z,1*sendCount_z,sendCount_z,sendbuf_z,Aq,N);
ScaLBL_D3Q19_Pack(13,dvcSendList_z,2*sendCount_z,sendCount_z,sendbuf_z,Aq,N);
ScaLBL_D3Q19_Pack(16,dvcSendList_z,3*sendCount_z,sendCount_z,sendbuf_z,Aq,N);
ScaLBL_D3Q19_Pack(17,dvcSendList_z,4*sendCount_z,sendCount_z,sendbuf_z,Aq,N);
ScaLBL_D3Q19_Pack(6 ,dvcSendList_z,5*sendCount_z,sendCount_z,sendbuf_z,Bq,N);
ScaLBL_D3Q19_Pack(12,dvcSendList_z,6*sendCount_z,sendCount_z,sendbuf_z,Bq,N);
ScaLBL_D3Q19_Pack(13,dvcSendList_z,7*sendCount_z,sendCount_z,sendbuf_z,Bq,N);
ScaLBL_D3Q19_Pack(16,dvcSendList_z,8*sendCount_z,sendCount_z,sendbuf_z,Bq,N);
ScaLBL_D3Q19_Pack(17,dvcSendList_z,9*sendCount_z,sendCount_z,sendbuf_z,Bq,N);
req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, 10*sendCount_z,rank_z,sendtag);
req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, 10*recvCount_Z,rank_Z,recvtag);
//...Packing for Z face(5,11,14,15,18)................................
ScaLBL_D3Q19_Pack(5 ,dvcSendList_Z,0*sendCount_Z,sendCount_Z,sendbuf_Z,Aq,N);
ScaLBL_D3Q19_Pack(11,dvcSendList_Z,1*sendCount_Z,sendCount_Z,sendbuf_Z,Aq,N);
ScaLBL_D3Q19_Pack(14,dvcSendList_Z,2*sendCount_Z,sendCount_Z,sendbuf_Z,Aq,N);
ScaLBL_D3Q19_Pack(15,dvcSendList_Z,3*sendCount_Z,sendCount_Z,sendbuf_Z,Aq,N);
ScaLBL_D3Q19_Pack(18,dvcSendList_Z,4*sendCount_Z,sendCount_Z,sendbuf_Z,Aq,N);
ScaLBL_D3Q19_Pack(5 ,dvcSendList_Z,5*sendCount_Z,sendCount_Z,sendbuf_Z,Bq,N);
ScaLBL_D3Q19_Pack(11,dvcSendList_Z,6*sendCount_Z,sendCount_Z,sendbuf_Z,Bq,N);
ScaLBL_D3Q19_Pack(14,dvcSendList_Z,7*sendCount_Z,sendCount_Z,sendbuf_Z,Bq,N);
ScaLBL_D3Q19_Pack(15,dvcSendList_Z,8*sendCount_Z,sendCount_Z,sendbuf_Z,Bq,N);
ScaLBL_D3Q19_Pack(18,dvcSendList_Z,9*sendCount_Z,sendCount_Z,sendbuf_Z,Bq,N);
req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, 10*sendCount_Z,rank_Z,sendtag);
req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, 10*recvCount_z,rank_z,recvtag);
//...Pack the xy edge (8)................................
ScaLBL_D3Q19_Pack(8,dvcSendList_xy,0*sendCount_xy,sendCount_xy,sendbuf_xy,Aq,N);
ScaLBL_D3Q19_Pack(8,dvcSendList_xy,1*sendCount_xy,sendCount_xy,sendbuf_xy,Bq,N);
req1[6] = MPI_COMM_SCALBL.Isend(sendbuf_xy, 2*sendCount_xy,rank_xy,sendtag);
req2[6] = MPI_COMM_SCALBL.Irecv(recvbuf_XY, 2*recvCount_XY,rank_XY,recvtag);
//...Pack the Xy edge (9)................................
ScaLBL_D3Q19_Pack(9,dvcSendList_Xy,0*sendCount_Xy,sendCount_Xy,sendbuf_Xy,Aq,N);
ScaLBL_D3Q19_Pack(9,dvcSendList_Xy,1*sendCount_Xy,sendCount_Xy,sendbuf_Xy,Bq,N);
req1[8] = MPI_COMM_SCALBL.Isend(sendbuf_Xy, 2*sendCount_Xy,rank_Xy,sendtag);
req2[8] = MPI_COMM_SCALBL.Irecv(recvbuf_xY, 2*recvCount_xY,rank_xY,recvtag);
//...Pack the xY edge (10)................................
ScaLBL_D3Q19_Pack(10,dvcSendList_xY,0*sendCount_xY,sendCount_xY,sendbuf_xY,Aq,N);
ScaLBL_D3Q19_Pack(10,dvcSendList_xY,1*sendCount_xY,sendCount_xY,sendbuf_xY,Bq,N);
req1[9] = MPI_COMM_SCALBL.Isend(sendbuf_xY, 2*sendCount_xY,rank_xY,sendtag);
req2[9] = MPI_COMM_SCALBL.Irecv(recvbuf_Xy, 2*recvCount_Xy,rank_Xy,recvtag);
//...Pack the XY edge (7)................................
ScaLBL_D3Q19_Pack(7,dvcSendList_XY,0*sendCount_XY,sendCount_XY,sendbuf_XY,Aq,N);
ScaLBL_D3Q19_Pack(7,dvcSendList_XY,1*sendCount_XY,sendCount_XY,sendbuf_XY,Bq,N);
req1[7] = MPI_COMM_SCALBL.Isend(sendbuf_XY, 2*sendCount_XY,rank_XY,sendtag);
req2[7] = MPI_COMM_SCALBL.Irecv(recvbuf_xy, 2*recvCount_xy,rank_xy,recvtag);
//...Pack the xz edge (12)................................
ScaLBL_D3Q19_Pack(12,dvcSendList_xz,0*sendCount_xz,sendCount_xz,sendbuf_xz,Aq,N);
ScaLBL_D3Q19_Pack(12,dvcSendList_xz,1*sendCount_xz,sendCount_xz,sendbuf_xz,Bq,N);
req1[10] = MPI_COMM_SCALBL.Isend(sendbuf_xz, 2*sendCount_xz,rank_xz,sendtag);
req2[10] = MPI_COMM_SCALBL.Irecv(recvbuf_XZ, 2*recvCount_XZ,rank_XZ,recvtag);
//...Pack the xZ edge (14)................................
ScaLBL_D3Q19_Pack(14,dvcSendList_xZ,0*sendCount_xZ,sendCount_xZ,sendbuf_xZ,Aq,N);
ScaLBL_D3Q19_Pack(14,dvcSendList_xZ,1*sendCount_xZ,sendCount_xZ,sendbuf_xZ,Bq,N);
req1[13] = MPI_COMM_SCALBL.Isend(sendbuf_xZ, 2*sendCount_xZ,rank_xZ,sendtag);
req2[13] = MPI_COMM_SCALBL.Irecv(recvbuf_Xz, 2*recvCount_Xz,rank_Xz,recvtag);
//...Pack the Xz edge (13)................................
ScaLBL_D3Q19_Pack(13,dvcSendList_Xz,0*sendCount_Xz,sendCount_Xz,sendbuf_Xz,Aq,N);
ScaLBL_D3Q19_Pack(13,dvcSendList_Xz,1*sendCount_Xz,sendCount_Xz,sendbuf_Xz,Bq,N);
req1[12] = MPI_COMM_SCALBL.Isend(sendbuf_Xz, 2*sendCount_Xz,rank_Xz,sendtag);
req2[12] = MPI_COMM_SCALBL.Irecv(recvbuf_xZ, 2*recvCount_xZ,rank_xZ,recvtag);
//...Pack the XZ edge (11)................................
ScaLBL_D3Q19_Pack(11,dvcSendList_XZ,0*sendCount_XZ,sendCount_XZ,sendbuf_XZ,Aq,N);
ScaLBL_D3Q19_Pack(11,dvcSendList_XZ,1*sendCount_XZ,sendCount_XZ,sendbuf_XZ,Bq,N);
req1[11] = MPI_COMM_SCALBL.Isend(sendbuf_XZ, 2*sendCount_XZ,rank_XZ,sendtag);
req2[11] = MPI_COMM_SCALBL.Irecv(recvbuf_xz, 2*recvCount_xz,rank_xz,recvtag);
//...Pack the yz edge (16)................................
ScaLBL_D3Q19_Pack(16,dvcSendList_yz,0*sendCount_yz,sendCount_yz,sendbuf_yz,Aq,N);
ScaLBL_D3Q19_Pack(16,dvcSendList_yz,1*sendCount_yz,sendCount_yz,sendbuf_yz,Bq,N);
req1[14] = MPI_COMM_SCALBL.Isend(sendbuf_yz, 2*sendCount_yz,rank_yz,sendtag);
req2[14] = MPI_COMM_SCALBL.Irecv(recvbuf_YZ, 2*recvCount_YZ,rank_YZ,recvtag);
//...Pack the yZ edge (18)................................
ScaLBL_D3Q19_Pack(18,dvcSendList_yZ,0*sendCount_yZ,sendCount_yZ,sendbuf_yZ,Aq,N);
ScaLBL_D3Q19_Pack(18,dvcSendList_yZ,1*sendCount_yZ,sendCount_yZ,sendbuf_yZ,Bq,N);
req1[17] = MPI_COMM_SCALBL.Isend(sendbuf_yZ, 2*sendCount_yZ,rank_yZ,sendtag);
req2[17] = MPI_COMM_SCALBL.Irecv(recvbuf_Yz, 2*recvCount_Yz,rank_Yz,recvtag);
//...Pack the Yz edge (17)................................
ScaLBL_D3Q19_Pack(17,dvcSendList_Yz,0*sendCount_Yz,sendCount_Yz,sendbuf_Yz,Aq,N);
ScaLBL_D3Q19_Pack(17,dvcSendList_Yz,1*sendCount_Yz,sendCount_Yz,sendbuf_Yz,Bq,N);
req1[16] = MPI_COMM_SCALBL.Isend(sendbuf_Yz, 2*sendCount_Yz,rank_Yz,sendtag);
req2[16] = MPI_COMM_SCALBL.Irecv(recvbuf_yZ, 2*recvCount_yZ,rank_yZ,recvtag);
//...Pack the YZ edge (15)................................
ScaLBL_D3Q19_Pack(15,dvcSendList_YZ,0*sendCount_YZ,sendCount_YZ,sendbuf_YZ,Aq,N);
ScaLBL_D3Q19_Pack(15,dvcSendList_YZ,1*sendCount_YZ,sendCount_YZ,sendbuf_YZ,Bq,N);
req1[15] = MPI_COMM_SCALBL.Isend(sendbuf_YZ, 2*sendCount_YZ,rank_YZ,sendtag);
req2[15] = MPI_COMM_SCALBL.Irecv(recvbuf_yz, 2*recvCount_yz,rank_yz,recvtag);
//...................................................................................
}
void ScaLBL_Communicator::BiRecvD3Q19AA(double *Aq, double *Bq){
// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
//...................................................................................
// Wait for completion of D3Q19 communication
MPI_COMM_SCALBL.waitAll(18,req1);
MPI_COMM_SCALBL.waitAll(18,req2);
ScaLBL_DeviceBarrier();
//...................................................................................
// NOTE: AA Routine writes to opposite
// Unpack the distributions on the device
//...................................................................................
//...Unpacking for x face(2,8,10,12,14)................................
ScaLBL_D3Q19_Unpack(2, dvcRecvDist_x,0*recvCount_x,recvCount_x,recvbuf_x,Aq,N);
ScaLBL_D3Q19_Unpack(8, dvcRecvDist_x,1*recvCount_x,recvCount_x,recvbuf_x,Aq,N);
ScaLBL_D3Q19_Unpack(10,dvcRecvDist_x,2*recvCount_x,recvCount_x,recvbuf_x,Aq,N);
ScaLBL_D3Q19_Unpack(12,dvcRecvDist_x,3*recvCount_x,recvCount_x,recvbuf_x,Aq,N);
ScaLBL_D3Q19_Unpack(14,dvcRecvDist_x,4*recvCount_x,recvCount_x,recvbuf_x,Aq,N);
ScaLBL_D3Q19_Unpack(2, dvcRecvDist_x,0*recvCount_x,recvCount_x,&recvbuf_x[5*recvCount_x],Bq,N);
ScaLBL_D3Q19_Unpack(8, dvcRecvDist_x,1*recvCount_x,recvCount_x,&recvbuf_x[5*recvCount_x],Bq,N);
ScaLBL_D3Q19_Unpack(10,dvcRecvDist_x,2*recvCount_x,recvCount_x,&recvbuf_x[5*recvCount_x],Bq,N);
ScaLBL_D3Q19_Unpack(12,dvcRecvDist_x,3*recvCount_x,recvCount_x,&recvbuf_x[5*recvCount_x],Bq,N);
ScaLBL_D3Q19_Unpack(14,dvcRecvDist_x,4*recvCount_x,recvCount_x,&recvbuf_x[5*recvCount_x],Bq,N);
//...................................................................................
//...Packing for X face(1,7,9,11,13)................................
ScaLBL_D3Q19_Unpack(1, dvcRecvDist_X,0*recvCount_X,recvCount_X,recvbuf_X,Aq,N);
ScaLBL_D3Q19_Unpack(7, dvcRecvDist_X,1*recvCount_X,recvCount_X,recvbuf_X,Aq,N);
ScaLBL_D3Q19_Unpack(9, dvcRecvDist_X,2*recvCount_X,recvCount_X,recvbuf_X,Aq,N);
ScaLBL_D3Q19_Unpack(11,dvcRecvDist_X,3*recvCount_X,recvCount_X,recvbuf_X,Aq,N);
ScaLBL_D3Q19_Unpack(13,dvcRecvDist_X,4*recvCount_X,recvCount_X,recvbuf_X,Aq,N);
ScaLBL_D3Q19_Unpack(1, dvcRecvDist_X,0*recvCount_X,recvCount_X,&recvbuf_X[5*recvCount_X],Bq,N);
ScaLBL_D3Q19_Unpack(7, dvcRecvDist_X,1*recvCount_X,recvCount_X,&recvbuf_X[5*recvCount_X],Bq,N);
ScaLBL_D3Q19_Unpack(9, dvcRecvDist_X,2*recvCount_X,recvCount_X,&recvbuf_X[5*recvCount_X],Bq,N);
ScaLBL_D3Q19_Unpack(11,dvcRecvDist_X,3*recvCount_X,recvCount_X,&recvbuf_X[5*recvCount_X],Bq,N);
ScaLBL_D3Q19_Unpack(13,dvcRecvDist_X,4*recvCount_X,recvCount_X,&recvbuf_X[5*recvCount_X],Bq,N);
//...................................................................................
//...Packing for y face(4,8,9,16,18).................................
ScaLBL_D3Q19_Unpack(4, dvcRecvDist_y,0*recvCount_y,recvCount_y,recvbuf_y,Aq,N);
ScaLBL_D3Q19_Unpack(8, dvcRecvDist_y,1*recvCount_y,recvCount_y,recvbuf_y,Aq,N);
ScaLBL_D3Q19_Unpack(9, dvcRecvDist_y,2*recvCount_y,recvCount_y,recvbuf_y,Aq,N);
ScaLBL_D3Q19_Unpack(16,dvcRecvDist_y,3*recvCount_y,recvCount_y,recvbuf_y,Aq,N);
ScaLBL_D3Q19_Unpack(18,dvcRecvDist_y,4*recvCount_y,recvCount_y,recvbuf_y,Aq,N);
ScaLBL_D3Q19_Unpack(4, dvcRecvDist_y,0*recvCount_y,recvCount_y,&recvbuf_y[5*recvCount_y],Bq,N);
ScaLBL_D3Q19_Unpack(8, dvcRecvDist_y,1*recvCount_y,recvCount_y,&recvbuf_y[5*recvCount_y],Bq,N);
ScaLBL_D3Q19_Unpack(9, dvcRecvDist_y,2*recvCount_y,recvCount_y,&recvbuf_y[5*recvCount_y],Bq,N);
ScaLBL_D3Q19_Unpack(16,dvcRecvDist_y,3*recvCount_y,recvCount_y,&recvbuf_y[5*recvCount_y],Bq,N);
ScaLBL_D3Q19_Unpack(18,dvcRecvDist_y,4*recvCount_y,recvCount_y,&recvbuf_y[5*recvCount_y],Bq,N);
//...................................................................................
//...Packing for Y face(3,7,10,15,17).................................
ScaLBL_D3Q19_Unpack(3, dvcRecvDist_Y,0*recvCount_Y,recvCount_Y,recvbuf_Y,Aq,N);
ScaLBL_D3Q19_Unpack(7, dvcRecvDist_Y,1*recvCount_Y,recvCount_Y,recvbuf_Y,Aq,N);
ScaLBL_D3Q19_Unpack(10,dvcRecvDist_Y,2*recvCount_Y,recvCount_Y,recvbuf_Y,Aq,N);
ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Y,3*recvCount_Y,recvCount_Y,recvbuf_Y,Aq,N);
ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Y,4*recvCount_Y,recvCount_Y,recvbuf_Y,Aq,N);
ScaLBL_D3Q19_Unpack(3, dvcRecvDist_Y,0*recvCount_Y,recvCount_Y,&recvbuf_Y[5*recvCount_Y],Bq,N);
ScaLBL_D3Q19_Unpack(7, dvcRecvDist_Y,1*recvCount_Y,recvCount_Y,&recvbuf_Y[5*recvCount_Y],Bq,N);
ScaLBL_D3Q19_Unpack(10,dvcRecvDist_Y,2*recvCount_Y,recvCount_Y,&recvbuf_Y[5*recvCount_Y],Bq,N);
ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Y,3*recvCount_Y,recvCount_Y,&recvbuf_Y[5*recvCount_Y],Bq,N);
ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Y,4*recvCount_Y,recvCount_Y,&recvbuf_Y[5*recvCount_Y],Bq,N);
//...................................................................................
//...Pack the xy edge (8)................................
ScaLBL_D3Q19_Unpack(8,dvcRecvDist_xy,0,recvCount_xy,recvbuf_xy,Aq,N);
ScaLBL_D3Q19_Unpack(8,dvcRecvDist_xy,0,recvCount_xy,&recvbuf_xy[recvCount_xy],Bq,N);
//...Pack the Xy edge (9)................................
ScaLBL_D3Q19_Unpack(9,dvcRecvDist_Xy,0,recvCount_Xy,recvbuf_Xy,Aq,N);
ScaLBL_D3Q19_Unpack(9,dvcRecvDist_Xy,0,recvCount_Xy,&recvbuf_Xy[recvCount_Xy],Bq,N);
//...Pack the xY edge (10)................................
ScaLBL_D3Q19_Unpack(10,dvcRecvDist_xY,0,recvCount_xY,recvbuf_xY,Aq,N);
ScaLBL_D3Q19_Unpack(10,dvcRecvDist_xY,0,recvCount_xY,&recvbuf_xY[recvCount_xY],Bq,N);
//...Pack the XY edge (7)................................
ScaLBL_D3Q19_Unpack(7,dvcRecvDist_XY,0,recvCount_XY,recvbuf_XY,Aq,N);
ScaLBL_D3Q19_Unpack(7,dvcRecvDist_XY,0,recvCount_XY,&recvbuf_XY[recvCount_XY],Bq,N);
if (BoundaryCondition > 0 && kproc == 0){
// don't unpack little z
//...Packing for Z face(5,11,14,15,18)................................
ScaLBL_D3Q19_Unpack(5, dvcRecvDist_Z,0*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N);
ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,1*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N);
ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N);
ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N);
ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N);
ScaLBL_D3Q19_Unpack(5, dvcRecvDist_Z,0*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N);
ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,1*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N);
ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N);
ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N);
ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N);
//...Pack the xZ edge (14)................................
ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,recvbuf_xZ,Aq,N);
ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,&recvbuf_xZ[recvCount_xZ],Bq,N);
//...Pack the XZ edge (11)................................
ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,recvbuf_XZ,Aq,N);
ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,&recvbuf_XZ[recvCount_XZ],Bq,N);
//...Pack the yZ edge (18)................................
ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,recvbuf_yZ,Aq,N);
ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,&recvbuf_yZ[recvCount_yZ],Bq,N);
//...Pack the YZ edge (15)................................
ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,recvbuf_YZ,Aq,N);
ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,&recvbuf_YZ[recvCount_YZ],Bq,N);
}
else if (BoundaryCondition > 0 && kproc == nprocz-1){
// don't unpack big Z
//...Packing for z face(6,12,13,16,17)................................
ScaLBL_D3Q19_Unpack(6, dvcRecvDist_z,0*recvCount_z,recvCount_z,recvbuf_z,Aq,N);
ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,1*recvCount_z,recvCount_z,recvbuf_z,Aq,N);
ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,Aq,N);
ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,recvbuf_z,Aq,N);
ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,recvbuf_z,Aq,N);
ScaLBL_D3Q19_Unpack(6, dvcRecvDist_z,0*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N);
ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,1*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N);
ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N);
ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N);
ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N);
//...Pack the xz edge (12)................................
ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,recvbuf_xz,Aq,N);
ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,&recvbuf_xz[recvCount_xz],Bq,N);
//...Pack the Xz edge (13)................................
ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,recvbuf_Xz,Aq,N);
ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,&recvbuf_Xz[recvCount_Xz],Bq,N);
//...Pack the yz edge (16)................................
ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,recvbuf_yz,Aq,N);
ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,&recvbuf_yz[recvCount_yz],Bq,N);
//...Pack the Yz edge (17)................................
ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,recvbuf_Yz,Aq,N);
ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,&recvbuf_Yz[recvCount_Yz],Bq,N);
}
else {
//...Packing for z face(6,12,13,16,17)................................
ScaLBL_D3Q19_Unpack(6, dvcRecvDist_z,0*recvCount_z,recvCount_z,recvbuf_z,Aq,N);
ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,1*recvCount_z,recvCount_z,recvbuf_z,Aq,N);
ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,Aq,N);
ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,recvbuf_z,Aq,N);
ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,recvbuf_z,Aq,N);
ScaLBL_D3Q19_Unpack(6, dvcRecvDist_z,0*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N);
ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,1*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N);
ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N);
ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N);
ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N);
//...Packing for Z face(5,11,14,15,18)................................
ScaLBL_D3Q19_Unpack(5, dvcRecvDist_Z,0*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N);
ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,1*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N);
ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N);
ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N);
ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N);
ScaLBL_D3Q19_Unpack(5, dvcRecvDist_Z,0*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N);
ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,1*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N);
ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N);
ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N);
ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N);
//...Pack the xZ edge (14)................................
ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,recvbuf_xZ,Aq,N);
ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,&recvbuf_xZ[recvCount_xZ],Bq,N);
//...Pack the XZ edge (11)................................
ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,recvbuf_XZ,Aq,N);
ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,&recvbuf_XZ[recvCount_XZ],Bq,N);
//...Pack the yZ edge (18)................................
ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,recvbuf_yZ,Aq,N);
ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,&recvbuf_yZ[recvCount_yZ],Bq,N);
//...Pack the YZ edge (15)................................
ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,recvbuf_YZ,Aq,N);
ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,&recvbuf_YZ[recvCount_YZ],Bq,N);
//...Pack the xz edge (12)................................
ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,recvbuf_xz,Aq,N);
ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,&recvbuf_xz[recvCount_xz],Bq,N);
//...Pack the Xz edge (13)................................
ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,recvbuf_Xz,Aq,N);
ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,&recvbuf_Xz[recvCount_Xz],Bq,N);
//...Pack the yz edge (16)................................
ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,recvbuf_yz,Aq,N);
ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,&recvbuf_yz[recvCount_yz],Bq,N);
//...Pack the Yz edge (17)................................
ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,recvbuf_Yz,Aq,N);
ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,&recvbuf_Yz[recvCount_Yz],Bq,N);
}
//...................................................................................
Lock=false; // unlock the communicator after communications complete
//...................................................................................
}
void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){
// Recieves halo and incorporates into D3Q19 based stencil gradient computation
@ -1110,6 +1472,7 @@ void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){
ScaLBL_Gradient_Unpack(1.0,-1,0,0,dvcRecvDist_x,0,recvCount_x,recvbuf_x,phi,grad,N);
ScaLBL_Gradient_Unpack(0.5,-1,-1,0,dvcRecvDist_x,recvCount_x,recvCount_x,recvbuf_x,phi,grad,N);
ScaLBL_Gradient_Unpack(0.5,-1,1,0,dvcRecvDist_x,2*recvCount_x,recvCount_x,recvbuf_x,phi,grad,N);
ScaLBL_Gradient_Unpack(0.5,-1,0,-1,dvcRecvDist_x,3*recvCount_x,recvCount_x,recvbuf_x,phi,grad,N);
ScaLBL_Gradient_Unpack(0.5,-1,0,1,dvcRecvDist_x,4*recvCount_x,recvCount_x,recvbuf_x,phi,grad,N);
//...................................................................................
//...Packing for X face(1,7,9,11,13)................................
@ -1176,6 +1539,106 @@ void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){
}
void ScaLBL_Communicator::SendD3Q7AA(double *Aq){
// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
if (Lock==true){
ERROR("ScaLBL Error (SendD3Q7): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?");
}
else{
Lock=true;
}
// assign tag of 19 to D3Q19 communication
sendtag = recvtag = 7;
ScaLBL_DeviceBarrier();
// Pack the distributions
//...Packing for x face(2,8,10,12,14)................................
ScaLBL_D3Q19_Pack(2,dvcSendList_x,0,sendCount_x,sendbuf_x,Aq,N);
req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, sendCount_x,rank_x,sendtag);
req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag);
//...Packing for X face(1,7,9,11,13)................................
ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,Aq,N);
req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, sendCount_X,rank_X,sendtag);
req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag);
//...Packing for y face(4,8,9,16,18).................................
ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,Aq,N);
req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, sendCount_y,rank_y,sendtag);
req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag);
//...Packing for Y face(3,7,10,15,17).................................
ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,Aq,N);
req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag);
req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag);
//...Packing for z face(6,12,13,16,17)................................
ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,Aq,N);
req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, sendCount_z,rank_z,sendtag);
req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag);
//...Packing for Z face(5,11,14,15,18)................................
ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,Aq,N);
req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, sendCount_Z,rank_Z,sendtag);
req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, recvCount_z,rank_z,recvtag);
//...................................................................................
}
void ScaLBL_Communicator::RecvD3Q7AA(double *Aq){
// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
//...................................................................................
// Wait for completion of D3Q19 communication
MPI_COMM_SCALBL.waitAll(6,req1);
MPI_COMM_SCALBL.waitAll(6,req2);
ScaLBL_DeviceBarrier();
//...................................................................................
// NOTE: AA Routine writes to opposite
// Unpack the distributions on the device
//...................................................................................
//...Unpacking for x face(2,8,10,12,14)................................
ScaLBL_D3Q7_Unpack(2,dvcRecvDist_x,0,recvCount_x,recvbuf_x,Aq,N);
//...................................................................................
//...Packing for X face(1,7,9,11,13)................................
ScaLBL_D3Q7_Unpack(1,dvcRecvDist_X,0,recvCount_X,recvbuf_X,Aq,N);
//...................................................................................
//...Packing for y face(4,8,9,16,18).................................
ScaLBL_D3Q7_Unpack(4,dvcRecvDist_y,0,recvCount_y,recvbuf_y,Aq,N);
//...................................................................................
//...Packing for Y face(3,7,10,15,17).................................
ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,0,recvCount_Y,recvbuf_Y,Aq,N);
//...................................................................................
if (BoundaryCondition > 0 && kproc == 0){
// don't unpack little z
//...Packing for Z face(5,11,14,15,18)................................
ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N);
}
else if (BoundaryCondition > 0 && kproc == nprocz-1){
// don't unpack big z
//...Packing for z face(6,12,13,16,17)................................
ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N);
}
else {
//...Packing for z face(6,12,13,16,17)................................
ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N);
//...Packing for Z face(5,11,14,15,18)................................
ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N);
}
//...................................................................................
Lock=false; // unlock the communicator after communications complete
//...................................................................................
}
void ScaLBL_Communicator::BiSendD3Q7AA(double *Aq, double *Bq){
// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
@ -1708,3 +2171,42 @@ void ScaLBL_Communicator::PrintD3Q19(){
delete [] TempBuffer;
}
void ScaLBL_Communicator::GreyscaleSC_BC_z(int *Map, double *DenA, double *DenB, double vA, double vB)
{
if (kproc == 0) {
// Set the density field on the z inlet
ScaLBL_GreyscaleSC_BC_z(dvcSendList_z, Map, DenA, DenB, vA, vB, sendCount_z);
}
}
void ScaLBL_Communicator::GreyscaleSC_BC_Z(int *Map, double *DenA, double *DenB, double vA, double vB)
{
if (kproc == nprocz-1){
// Set the density field on the Z outlet
ScaLBL_GreyscaleSC_BC_Z(dvcSendList_Z, Map, DenA, DenB, vA, vB, sendCount_Z);
}
}
void ScaLBL_Communicator::GreyscaleSC_Pressure_BC_z(int *neighborList, double *fqA, double *fqB, double dinA, double dinB, int time)
{
if (kproc == 0) {
if (time%2==0){
ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z(dvcSendList_z, fqA, fqB, dinA, dinB, sendCount_z, N);
}
else{
ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z(neighborList, dvcSendList_z, fqA, fqB, dinA, dinB, sendCount_z, N);
}
}
}
void ScaLBL_Communicator::GreyscaleSC_Pressure_BC_Z(int *neighborList, double *fqA, double *fqB, double doutA, double doutB, int time)
{
if (kproc == nprocz-1){
if (time%2==0){
ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z(dvcSendList_Z, fqA, fqB, doutA, doutB, sendCount_Z, N);
}
else{
ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(neighborList, dvcSendList_Z, fqA, fqB, doutA, doutB, sendCount_Z, N);
}
}
}

127
common/ScaLBL.h
View File

@ -56,7 +56,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int
extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz);
// GREYSCALE MODEL
// GREYSCALE MODEL (Single-component)
extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *Dist, int Np, double Den);
@ -72,6 +72,119 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int
extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,
double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,
double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,
double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
// GREYSCALE FREE-ENERGY MODEL (Two-component)
extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFE(double *dist, double *Aq, double *Bq, double *Den,
double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np,
double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz,
double *Poros,double *Perm, double *Velocity,double *Pressure);
extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFE(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,
double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np,
double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz,
double *Poros,double *Perm, double *Velocity,double *Pressure);
extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFEChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np,
double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB,
double Gx, double Gy, double Gz,
double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap);
extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFEChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np,
double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB,
double Gx, double Gy, double Gz,
double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap);
extern "C" void ScaLBL_D3Q7_GreyscaleFE_Init(double *Den, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_GreyscaleFE_IMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEDensity(int *NeighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEPhi(int *NeighborList, double *Cq, double *Phi, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEPhi(double *Cq, double *Phi, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_GreyscaleFE_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_GreyscaleFE_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_GreyscaleFE_Pressure(double *dist, double *Den, double *Porosity,double *Velocity,
double *Pressure, double rhoA,double rhoB, int Np);
extern "C" void ScaLBL_D3Q19_GreyscaleFE_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap,
double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np);
// GREYSCALE SHAN-CHEN MODEL (Two-component)
extern "C" void ScaLBL_D3Q19_GreyscaleSC_Init(int *Map, double *distA, double *distB, double *DenA, double *DenB, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(int *NeighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, int *Mpa, double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB,
double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure,
double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,
int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(int *Map,double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB,
double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure,
double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,
int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(int *neighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB,
double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure,
double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,
int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB,
double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure,
double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,
int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Map, double *Den, double *DenGrad, int strideY, int strideZ,int start, int finish, int Np);
extern "C" void ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count);
extern "C" void ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count);
extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z(int *list, double *distA, double *distB, double dinA, double dinB, int count, int N);
extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z(int *list, double *distA, double *distB, double doutA, double doutB, int count, int N);
extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z(int *neighborList, int *list, double *distA, double *distB, double dinA, double dinB, int count, int N);
extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *distA, double *distB, double doutA, double doutB, int count, int N);
// GREYSCALE COLOR MODEL (Two-component)
//extern "C" void ScaLBL_D3Q19_GreyscaleColor_Init(double *dist, double *Porosity, int Np);
//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den,
// double *ColorGrad,double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel,
// double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta,
// double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
//
//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den,
// double *ColorGrad,double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel,
// double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta,
// double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den,
double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel,
double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta,
double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den,
double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel,
double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta,
double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
// MRT MODEL
extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx,
@ -185,6 +298,12 @@ public:
int MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, signed char *id, int Np);
void SendD3Q19AA(double *dist);
void RecvD3Q19AA(double *dist);
void BiSendD3Q19AA(double *Aq, double *Bq);
void BiRecvD3Q19AA(double *Aq, double *Bq);
// void BiSendD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
// void BiRecvD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
void SendD3Q7AA(double *Aq);
void RecvD3Q7AA(double *Aq);
void BiSendD3Q7AA(double *Aq, double *Bq);
void BiRecvD3Q7AA(double *Aq, double *Bq);
void TriSendD3Q7AA(double *Aq, double *Bq, double *Cq);
@ -202,6 +321,12 @@ public:
void D3Q19_Reflection_BC_z(double *fq);
void D3Q19_Reflection_BC_Z(double *fq);
double D3Q19_Flux_BC_z(int *neighborList, double *fq, double flux, int time);
void GreyscaleSC_BC_z(int *Map, double *DenA, double *DenB, double vA, double vB);
void GreyscaleSC_BC_Z(int *Map, double *DenA, double *DenB, double vA, double vB);
void GreyscaleSC_Pressure_BC_z(int *neighborList, double *fqA, double *fqB, double dinA, double dinB, int time);
void GreyscaleSC_Pressure_BC_Z(int *neighborList, double *fqA, double *fqB, double doutA, double doutB, int time);
// void TestSendD3Q19(double *f_even, double *f_odd);
// void TestRecvD3Q19(double *f_even, double *f_odd);
// Debugging and unit testing functions
void PrintD3Q19();

27
cpu/D3Q19.cpp
View File

@ -84,33 +84,6 @@ extern "C" void ScaLBL_D3Q19_Init(double *dist, int Np)
}
}
extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *dist, int Np, double Den)
{
int n;
for (n=0; n<Np; n++){
dist[n] = Den - 0.6666666666666667;
dist[Np+n] = 0.055555555555555555; //double(100*n)+1.f;
dist[2*Np+n] = 0.055555555555555555; //double(100*n)+2.f;
dist[3*Np+n] = 0.055555555555555555; //double(100*n)+3.f;
dist[4*Np+n] = 0.055555555555555555; //double(100*n)+4.f;
dist[5*Np+n] = 0.055555555555555555; //double(100*n)+5.f;
dist[6*Np+n] = 0.055555555555555555; //double(100*n)+6.f;
dist[7*Np+n] = 0.0277777777777778; //double(100*n)+7.f;
dist[8*Np+n] = 0.0277777777777778; //double(100*n)+8.f;
dist[9*Np+n] = 0.0277777777777778; //double(100*n)+9.f;
dist[10*Np+n] = 0.0277777777777778; //double(100*n)+10.f;
dist[11*Np+n] = 0.0277777777777778; //double(100*n)+11.f;
dist[12*Np+n] = 0.0277777777777778; //double(100*n)+12.f;
dist[13*Np+n] = 0.0277777777777778; //double(100*n)+13.f;
dist[14*Np+n] = 0.0277777777777778; //double(100*n)+14.f;
dist[15*Np+n] = 0.0277777777777778; //double(100*n)+15.f;
dist[16*Np+n] = 0.0277777777777778; //double(100*n)+16.f;
dist[17*Np+n] = 0.0277777777777778; //double(100*n)+17.f;
dist[18*Np+n] = 0.0277777777777778; //double(100*n)+18.f;
}
}
//*************************************************************************
extern "C" void ScaLBL_D3Q19_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz)
{

1076
cpu/Greyscale.cpp
View File

File diff suppressed because it is too large Load Diff

1366
cpu/GreyscaleColor.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

3113
cpu/GreyscaleFE.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

3625
cpu/GreyscaleSC.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

38
gpu/D3Q19.cu
View File

@ -267,37 +267,6 @@ __global__ void dvc_ScaLBL_D3Q19_Init(double *dist, int Np)
}
}
__global__ void dvc_ScaLBL_D3Q19_GreyIMRT_Init(double *dist, int Np, double Den)
{
int n;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
if (n<Np ){
dist[n] = Den - 0.6666666666666667;
dist[Np+n] = 0.055555555555555555; //double(100*n)+1.f;
dist[2*Np+n] = 0.055555555555555555; //double(100*n)+2.f;
dist[3*Np+n] = 0.055555555555555555; //double(100*n)+3.f;
dist[4*Np+n] = 0.055555555555555555; //double(100*n)+4.f;
dist[5*Np+n] = 0.055555555555555555; //double(100*n)+5.f;
dist[6*Np+n] = 0.055555555555555555; //double(100*n)+6.f;
dist[7*Np+n] = 0.0277777777777778; //double(100*n)+7.f;
dist[8*Np+n] = 0.0277777777777778; //double(100*n)+8.f;
dist[9*Np+n] = 0.0277777777777778; //double(100*n)+9.f;
dist[10*Np+n] = 0.0277777777777778; //double(100*n)+10.f;
dist[11*Np+n] = 0.0277777777777778; //double(100*n)+11.f;
dist[12*Np+n] = 0.0277777777777778; //double(100*n)+12.f;
dist[13*Np+n] = 0.0277777777777778; //double(100*n)+13.f;
dist[14*Np+n] = 0.0277777777777778; //double(100*n)+14.f;
dist[15*Np+n] = 0.0277777777777778; //double(100*n)+15.f;
dist[16*Np+n] = 0.0277777777777778; //double(100*n)+16.f;
dist[17*Np+n] = 0.0277777777777778; //double(100*n)+17.f;
dist[18*Np+n] = 0.0277777777777778; //double(100*n)+18.f;
}
}
}
//*************************************************************************
__global__ void dvc_ScaLBL_D3Q19_Swap_Compact(int *neighborList, double *disteven, double *distodd, int Np, int q){
int n,nn;
@ -2395,13 +2364,6 @@ extern "C" void ScaLBL_D3Q19_Init(double *dist, int Np){
}
}
extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *dist, int Np, double Den){
dvc_ScaLBL_D3Q19_GreyIMRT_Init<<<NBLOCKS,NTHREADS >>>(dist, Np, Den);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q19_GreyIMRT_Init: %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q19_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz){
dvc_ScaLBL_D3Q19_Swap<<<NBLOCKS,NTHREADS >>>(ID, disteven, distodd, Nx, Ny, Nz);

1124
gpu/Greyscale.cu
View File

File diff suppressed because it is too large Load Diff

2993
gpu/GreyscaleColor.cu Normal file
View File

File diff suppressed because it is too large Load Diff

3338
gpu/GreyscaleFE.cu Normal file
View File

File diff suppressed because it is too large Load Diff

3819
gpu/GreyscaleSC.cu Normal file
View File

File diff suppressed because it is too large Load Diff

2444
models/GreyscaleColorModel.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

97
models/GreyscaleColorModel.h Normal file
View File

@ -0,0 +1,97 @@
/*
Implementation of color lattice boltzmann model
*/
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <iostream>
#include <exception>
#include <stdexcept>
#include <fstream>
#include "common/Communication.h"
#include "analysis/TwoPhase.h"
#include "analysis/runAnalysis.h"
#include "common/MPI.h"
#include "ProfilerApp.h"
#include "threadpool/thread_pool.h"
class ScaLBL_GreyscaleColorModel{
public:
ScaLBL_GreyscaleColorModel(int RANK, int NP, const Utilities::MPI& COMM);
~ScaLBL_GreyscaleColorModel();
// functions in they should be run
void ReadParams(string filename);
void ReadParams(std::shared_ptr<Database> db0);
void SetDomain();
void ReadInput();
void Create();
void Initialize();
void Run();
void WriteDebug();
bool Restart,pBC;
bool REVERSE_FLOW_DIRECTION;
int timestep,timestepMax;
int BoundaryCondition;
double tauA,tauB,rhoA,rhoB,alpha,beta;
double tauA_eff,tauB_eff;
double Fx,Fy,Fz,flux;
double din,dout,inletA,inletB,outletA,outletB;
double GreyPorosity;
bool greyMode;//run greyColor model if true
int Nx,Ny,Nz,N,Np;
int rank,nprocx,nprocy,nprocz,nprocs;
double Lx,Ly,Lz;
std::shared_ptr<Domain> Dm; // this domain is for analysis
std::shared_ptr<Domain> Mask; // this domain is for lbm
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm;
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm_Regular;
//std::shared_ptr<TwoPhase> Averages;
std::shared_ptr<SubPhase> Averages;
// input database
std::shared_ptr<Database> db;
std::shared_ptr<Database> domain_db;
std::shared_ptr<Database> greyscaleColor_db;
std::shared_ptr<Database> analysis_db;
std::shared_ptr<Database> vis_db;
IntArray Map;
signed char *id;
int *NeighborList;
int *dvcMap;
double *fq, *Aq, *Bq;
double *Den, *Phi;
//double *GreySolidPhi; //Model 2 & 3
double *GreySolidGrad;//Model 1
//double *ColorGrad;
double *Velocity;
double *Pressure;
double *Porosity_dvc;
double *Permeability_dvc;
private:
Utilities::MPI comm;
int dist_mem_size;
int neighborSize;
// filenames
char LocalRankString[8];
char LocalRankFilename[40];
char LocalRestartFile[40];
//int rank,nprocs;
void LoadParams(std::shared_ptr<Database> db0);
void AssignComponentLabels();
void AssignGreySolidLabels();
void AssignGreyPoroPermLabels();
double ImageInit(std::string filename);
double MorphInit(const double beta, const double morph_delta);
double SeedPhaseField(const double seed_water_in_oil);
double MorphOpenConnected(double target_volume_change);
};

1247
models/GreyscaleFEModel.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

108
models/GreyscaleFEModel.h Normal file
View File

@ -0,0 +1,108 @@
/*
Implementation of multicomponent greyscale free-energy based lattice boltzmann model
*/
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <iostream>
#include <exception>
#include <stdexcept>
#include <fstream>
#include "common/Communication.h"
#include "common/MPI.h"
#include "common/Database.h"
#include "common/ScaLBL.h"
#include "ProfilerApp.h"
#include "threadpool/thread_pool.h"
class ScaLBL_GreyscaleFEModel{
public:
ScaLBL_GreyscaleFEModel(int RANK, int NP, MPI_Comm COMM);
~ScaLBL_GreyscaleFEModel();
// functions in they should be run
void ReadParams(string filename);
void ReadParams(std::shared_ptr<Database> db0);
void SetDomain();
void ReadInput();
void Create();
void Initialize();
void Run();
void WriteDebug();
void VelocityField();
bool Restart,pBC;
int timestep,timestepMax;
int BoundaryCondition;
double tauA,tauB;
double tauA_eff,tauB_eff;
double rhoA,rhoB;
double tolerance;
double Fx,Fy,Fz,flux;
double din,dout;
double dp;//solid particle diameter, unit in voxel
double GreyPorosity;
//double Gsc;
double gamma;
double kappaA,kappaB;
double lambdaA,lambdaB;
int Nx,Ny,Nz,N,Np;
int rank,nprocx,nprocy,nprocz,nprocs;
double Lx,Ly,Lz;
std::shared_ptr<Domain> Dm; // this domain is for analysis
std::shared_ptr<Domain> Mask; // this domain is for lbm
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm;
// input database
std::shared_ptr<Database> db;
std::shared_ptr<Database> domain_db;
std::shared_ptr<Database> greyscaleFE_db;
std::shared_ptr<Database> analysis_db;
std::shared_ptr<Database> vis_db;
signed char *id;
int *NeighborList;
//double *fq,*Aq,*Bq;
double *fq,*Cq;
double *Den;
double *Permeability;//grey voxel permeability
double *Porosity;
double *Velocity;
double *SolidForce;
double *Pressure_dvc;
double *PressureGrad;// gradiant of pressure
double *PressTensor;//pressure tensor
double *PressTensorGrad;// gradient of pressure tensor
double *Phi;
double *PhiLap;//laplacian of phase field phi
// double *DenGradA;
// double *DenGradB;
// double *DenLapA;
// double *DenLapB;
IntArray Map;
DoubleArray SignDist;
DoubleArray Velocity_x;
DoubleArray Velocity_y;
DoubleArray Velocity_z;
DoubleArray PorosityMap;
DoubleArray Pressure;
private:
MPI_Comm comm;
int dist_mem_size;
int neighborSize;
// filenames
char LocalRankString[8];
char LocalRankFilename[40];
char LocalRestartFile[40];
void AssignComponentLabels(double *Porosity, double *Permeablity, double *SolidPotential);
void AssignSolidForce(double *SolidPotential, double *SolidForce);
void Density_and_Phase_Init();
};

88
models/GreyscaleModel.cpp
View File

@ -44,7 +44,7 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){
din=dout=1.0;
flux=0.0;
dp = 10.0; //unit of 'dp': voxel
CollisionType = 1; //1: IMRT; 2: BGK
CollisionType = 1; //1: IMRT; 2: BGK; 3: MRT
// ---------------------- Greyscale Model parameters -----------------------//
if (greyscale_db->keyExists( "timestepMax" )){
@ -84,6 +84,9 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){
if (collision == "BGK"){
CollisionType=2;
}
else if (collision == "MRT"){
CollisionType=3;
}
// ------------------------------------------------------------------------//
//------------------------ Other Domain parameters ------------------------//
@ -199,9 +202,9 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm
for (int idx=0; idx<NLABELS; idx++) label_count[idx]=0;
for (int k=1;k<Nz-1;k++){
for (int j=1;j<Ny-1;j++){
for (int i=1;i<Nx-1;i++){
for (int k=0;k<Nz;k++){
for (int j=0;j<Ny;j++){
for (int i=0;i<Nx;i++){
int n = k*Nx*Ny+j*Nx+i;
VALUE=id[n];
// Assign the affinity from the paired list
@ -230,9 +233,9 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm
if (NLABELS != PermeabilityList.size()){
ERROR("Error: ComponentLabels and PermeabilityList must be the same length! \n");
}
for (int k=1;k<Nz-1;k++){
for (int j=1;j<Ny-1;j++){
for (int i=1;i<Nx-1;i++){
for (int k=0;k<Nz;k++){
for (int j=0;j<Ny;j++){
for (int i=0;i<Nx;i++){
int n = k*Nx*Ny+j*Nx+i;
VALUE=id[n];
// Assign the affinity from the paired list
@ -323,7 +326,6 @@ void ScaLBL_GreyscaleModel::Create(){
neighborSize=18*(Np*sizeof(int));
//...........................................................................
ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize);
ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np);
ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np);
ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np);
@ -331,47 +333,19 @@ void ScaLBL_GreyscaleModel::Create(){
ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np);
//...........................................................................
// Update GPU data structures
if (rank==0) printf ("Setting up device map and neighbor list \n");
if (rank==0) printf ("Setting up device neighbor list \n");
fflush(stdout);
int *TmpMap;
TmpMap=new int[Np];
for (int k=1; k<Nz-1; k++){
for (int j=1; j<Ny-1; j++){
for (int i=1; i<Nx-1; i++){
int idx=Map(i,j,k);
if (!(idx < 0))
TmpMap[idx] = k*Nx*Ny+j*Nx+i;
}
}
}
// check that TmpMap is valid
for (int idx=0; idx<ScaLBL_Comm->LastExterior(); idx++){
int n = TmpMap[idx];
if (n > Nx*Ny*Nz){
printf("Bad value! idx=%i \n");
TmpMap[idx] = Nx*Ny*Nz-1;
}
}
for (int idx=ScaLBL_Comm->FirstInterior(); idx<ScaLBL_Comm->LastInterior(); idx++){
int n = TmpMap[idx];
if (n > Nx*Ny*Nz){
printf("Bad value! idx=%i \n");
TmpMap[idx] = Nx*Ny*Nz-1;
}
}
ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np);
ScaLBL_DeviceBarrier();
delete [] TmpMap;
// copy the neighbor list
ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
// initialize phi based on PhaseLabel (include solid component labels)
double *Poros, *Perm;
Poros = new double[Np];
Perm = new double[Np];
Perm = new double[Np];
AssignComponentLabels(Poros,Perm);
ScaLBL_CopyToDevice(Porosity, Poros, Np*sizeof(double));
ScaLBL_CopyToDevice(Permeability, Perm, Np*sizeof(double));
delete [] Poros;
delete [] Perm;
}
@ -388,6 +362,10 @@ void ScaLBL_GreyscaleModel::Initialize(){
ScaLBL_D3Q19_Init(fq, Np);
if (rank==0) printf("Collision model: BGK.\n");
}
else if (CollisionType==3){
ScaLBL_D3Q19_Init(fq, Np);
if (rank==0) printf("Collision model: MRT.\n");
}
else{
if (rank==0) printf("Unknown collison type! IMRT collision is used.\n");
ScaLBL_D3Q19_GreyIMRT_Init(fq, Np, Den);
@ -469,6 +447,9 @@ void ScaLBL_GreyscaleModel::Run(){
case 2:
ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc);
break;
case 3:
ScaLBL_D3Q19_AAodd_Greyscale_MRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
break;
default:
ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
break;
@ -487,6 +468,9 @@ void ScaLBL_GreyscaleModel::Run(){
case 2:
ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc);
break;
case 3:
ScaLBL_D3Q19_AAodd_Greyscale_MRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
break;
default:
ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
break;
@ -503,6 +487,9 @@ void ScaLBL_GreyscaleModel::Run(){
case 2:
ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc);
break;
case 3:
ScaLBL_D3Q19_AAeven_Greyscale_MRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
break;
default:
ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
break;
@ -521,6 +508,9 @@ void ScaLBL_GreyscaleModel::Run(){
case 2:
ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc);
break;
case 3:
ScaLBL_D3Q19_AAeven_Greyscale_MRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
break;
default:
ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
break;
@ -594,15 +584,15 @@ void ScaLBL_GreyscaleModel::Run(){
}
}
}
//MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
//MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
//MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
//MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
vax = sumReduce( Mask->Comm, vax_loc);
vay = sumReduce( Mask->Comm, vay_loc);
vaz = sumReduce( Mask->Comm, vaz_loc);
count = sumReduce( Mask->Comm, count_loc);
// Disable the this, if the one below does not work, then enable this
//vax = sumReduce( Mask->Comm, vax_loc);
//vay = sumReduce( Mask->Comm, vay_loc);
//vaz = sumReduce( Mask->Comm, vaz_loc);
//count = sumReduce( Mask->Comm, count_loc);
vax = Mask->Comm.sumReduce( vax_loc );
vay = Mask->Comm.sumReduce( vay_loc );
vaz = Mask->Comm.sumReduce( vaz_loc );
count = Mask->Comm.sumReduce( count_loc );
vax /= count;
vay /= count;

1
models/GreyscaleModel.h
View File

@ -62,7 +62,6 @@ public:
signed char *id;
int *NeighborList;
int *dvcMap;
double *fq;
double *Permeability;//grey voxel permeability
double *Porosity;

1433
models/GreyscaleSCModel.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

103
models/GreyscaleSCModel.h Normal file
View File

@ -0,0 +1,103 @@
/*
Implementation of color lattice boltzmann model
*/
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <iostream>
#include <exception>
#include <stdexcept>
#include <fstream>
#include "common/Communication.h"
#include "common/MPI.h"
#include "common/Database.h"
#include "common/ScaLBL.h"
#include "ProfilerApp.h"
#include "threadpool/thread_pool.h"
class ScaLBL_GreyscaleSCModel{
public:
ScaLBL_GreyscaleSCModel(int RANK, int NP, MPI_Comm COMM);
~ScaLBL_GreyscaleSCModel();
// functions in they should be run
void ReadParams(string filename);
void ReadParams(std::shared_ptr<Database> db0);
void SetDomain();
void ReadInput();
void Create();
void Initialize();
void Run();
void WriteDebug();
void WriteOutput();
bool Restart,pBC;
int timestep,timestepMax;
int BoundaryCondition;
int CollisionType;
double tauA,tauB;
double tauA_eff,tauB_eff;
double Gsc;
double rhoA,rhoB;
double rhoA_minor,rhoB_minor;//dissolved density
double tolerance;
double Fx,Fy,Fz;
double fluxA,fluxB;
double dinA,doutA;
double dinB,doutB;
double GreyPorosity;
int Nx,Ny,Nz,N,Np;
int rank,nprocx,nprocy,nprocz,nprocs;
double Lx,Ly,Lz;
std::shared_ptr<Domain> Dm; // this domain is for analysis
std::shared_ptr<Domain> Mask; // this domain is for lbm
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm;
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm_Regular;
// input database
std::shared_ptr<Database> db;
std::shared_ptr<Database> domain_db;
std::shared_ptr<Database> greyscaleSC_db;
std::shared_ptr<Database> analysis_db;
std::shared_ptr<Database> vis_db;
signed char *id;
int *NeighborList;
int *dvcMap;
double *fqA, *fqB;
double *Permeability;//grey voxel permeability
//double relPermA,relPermB;//grey voxel relperm
double *Porosity;
double *Velocity;
double *Pressure_dvc;
double *DenA, *DenB;
double *DenGradA,*DenGradB;
double *SolidForceA,*SolidForceB;
IntArray Map;
DoubleArray SignDist;
DoubleArray Velocity_x;
DoubleArray Velocity_y;
DoubleArray Velocity_z;
DoubleArray PorosityMap;
DoubleArray Pressure;
DoubleArray DenA_data;
DoubleArray DenB_data;
private:
MPI_Comm comm;
int dist_mem_size;
int neighborSize;
// filenames
char LocalRankString[8];
char LocalRankFilename[40];
char LocalRestartFile[40];
void AssignGreyscaleAndSolidLabels();
void Density_Init();
};

3
tests/CMakeLists.txt
View File

@ -4,6 +4,9 @@
ADD_LBPM_EXECUTABLE( lbpm_color_simulator )
ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator )
ADD_LBPM_EXECUTABLE( lbpm_greyscale_simulator )
ADD_LBPM_EXECUTABLE( lbpm_greyscaleFE_simulator )
ADD_LBPM_EXECUTABLE( lbpm_greyscaleSC_simulator )
ADD_LBPM_EXECUTABLE( lbpm_greyscaleColor_simulator )
#ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator )
#ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator )
ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator )

58
tests/lbpm_greyscaleColor_simulator.cpp Normal file
View File

@ -0,0 +1,58 @@
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <iostream>
#include <exception>
#include <stdexcept>
#include <fstream>
#include "common/ScaLBL.h"
#include "common/Communication.h"
#include "common/MPI.h"
#include "models/GreyscaleColorModel.h"
//#define WRITE_SURFACES
using namespace std;
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
{
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
if (rank == 0){
printf("****************************************\n");
printf("Running Greyscale Two-Phase Calculation \n");
printf("****************************************\n");
}
// Initialize compute device
int device=ScaLBL_SetDevice(rank);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
ScaLBL_GreyscaleColorModel GreyscaleColor(rank,nprocs,comm);
auto filename = argv[1];
GreyscaleColor.ReadParams(filename);
GreyscaleColor.SetDomain(); // this reads in the domain
GreyscaleColor.ReadInput();
GreyscaleColor.Create(); // creating the model will create data structure to match the pore structure and allocate variables
GreyscaleColor.Initialize(); // initializing the model will set initial conditions for variables
GreyscaleColor.Run();
GreyscaleColor.WriteDebug();
}
// ****************************************************
MPI_Barrier(comm);
MPI_Finalize();
// ****************************************************
}

59
tests/lbpm_greyscaleFE_simulator.cpp Normal file
View File

@ -0,0 +1,59 @@
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <iostream>
#include <exception>
#include <stdexcept>
#include <fstream>
#include "common/ScaLBL.h"
#include "common/Communication.h"
#include "common/MPI.h"
#include "models/GreyscaleFEModel.h"
//#define WRITE_SURFACES
using namespace std;
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
{
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
if (rank == 0){
printf("****************************************\n");
printf("Running Greyscale Two-Phase Calculation \n");
printf("****************************************\n");
}
// Initialize compute device
int device=ScaLBL_SetDevice(rank);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
ScaLBL_GreyscaleFEModel GreyscaleFE(rank,nprocs,comm);
auto filename = argv[1];
GreyscaleFE.ReadParams(filename);
GreyscaleFE.SetDomain(); // this reads in the domain
GreyscaleFE.ReadInput();
GreyscaleFE.Create(); // creating the model will create data structure to match the pore structure and allocate variables
GreyscaleFE.Initialize(); // initializing the model will set initial conditions for variables
GreyscaleFE.Run();
//GreyscaleFE.VelocityField();
GreyscaleFE.WriteDebug();
}
// ****************************************************
MPI_Barrier(comm);
MPI_Finalize();
// ****************************************************
}

58
tests/lbpm_greyscaleSC_simulator.cpp Normal file
View File

@ -0,0 +1,58 @@
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <iostream>
#include <exception>
#include <stdexcept>
#include <fstream>
#include "common/ScaLBL.h"
#include "common/Communication.h"
#include "common/MPI.h"
#include "models/GreyscaleSCModel.h"
//#define WRITE_SURFACES
using namespace std;
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
{
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
if (rank == 0){
printf("****************************************\n");
printf("Running Greyscale Two-Phase Calculation \n");
printf("****************************************\n");
}
// Initialize compute device
int device=ScaLBL_SetDevice(rank);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
ScaLBL_GreyscaleSCModel GreyscaleSC(rank,nprocs,comm);
auto filename = argv[1];
GreyscaleSC.ReadParams(filename);
GreyscaleSC.SetDomain(); // this reads in the domain
GreyscaleSC.ReadInput();
GreyscaleSC.Create(); // creating the model will create data structure to match the pore structure and allocate variables
GreyscaleSC.Initialize(); // initializing the model will set initial conditions for variables
GreyscaleSC.Run();
GreyscaleSC.WriteDebug();
}
// ****************************************************
MPI_Barrier(comm);
MPI_Finalize();
// ****************************************************
}

2
tests/lbpm_greyscale_simulator.cpp
View File

@ -53,7 +53,7 @@ int main(int argc, char **argv)
Greyscale.Create(); // creating the model will create data structure to match the pore structure and allocate variables
Greyscale.Initialize(); // initializing the model will set initial conditions for variables
Greyscale.Run();
//Greyscale.VelocityField();
Greyscale.VelocityField();
//Greyscale.WriteDebug();
}
// ****************************************************