Merge branch 'Crusher' of github.com:JamesEMcClure/LBPM-WIA into Crusher

sample_scripts/configure_crusher_cpu

@@ -0,0 +1,47 @@
+#module load cmake/3.21.3
+#module load PrgEnv-gnu
+module load PrgEnv-amd
+module load rocm/4.5.0
+module load cray-mpich
+module load cray-hdf5-parallel
+#module load craype-accel-amd-gfx908
+
+## These must be set before compiling so the executable picks up GTL
+export PE_MPICH_GTL_DIR_amd_gfx90a="-L${CRAY_MPICH_ROOTDIR}/gtl/lib"
+export PE_MPICH_GTL_LIBS_amd_gfx90a="-lmpi_gtl_hsa"
+
+export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}
+
+# Need a new version of cmake
+export CMAKE_DIR=/gpfs/alpine/csc380/proj-shared/LBPM/cmake-3.21.0/bin
+
+#-I${MPICH_DIR}/include
+#-L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa
+
+HIPFLAGS = --amdgpu-target=gfx90a 
+
+# configure
+rm -rf CMake*
+${CMAKE_DIR}/cmake                             \
+    -D CMAKE_BUILD_TYPE:STRING=Release          \
+    -D CMAKE_C_COMPILER:PATH=cc               \
+    -D CMAKE_CXX_COMPILER:PATH=CC               \
+    -D CMAKE_CXX_STANDARD=14                    \
+    -D DISABLE_GOLD:BOOL=TRUE                   \
+    -D DISABLE_LTO:BOOL=TRUE                    \
+    -D CMAKE_C_FLAGS="-L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa -I${HDF5_DIR}/include"  \
+    -D CMAKE_CXX_FLAGS="-L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa  -I${HDF5_DIR}/include" \
+    -D LINK_LIBRARIES="${ROCM_PATH}/lib/libamdhip64.so;${CRAY_MPICH_ROOTDIR}/gtl/lib/libmpi_gtl_hsa.so" \
+    -D USE_HIP=0                                \
+        -D CMAKE_HIP_COMPILER_TOOLKIT_ROOT=$ROCM_PATH/hip \
+    -D USE_MPI=1                                \
+    -D MPI_SKIP_SEARCH=1                        \
+    -D MPIEXEC="srun"                           \
+    -D USE_HDF5=1                               \
+        -D HDF5_DIRECTORY="${HDF5_DIR}"         \
+    -D USE_SILO=0                               \
+    -D USE_TIMER=0			        \
+    -D USE_DOXYGEN:BOOL=false		        \
+    ~/LBPM-WIA
+
+

sample_scripts/configure_crusher_hip

@@ -18,7 +18,11 @@ export CMAKE_DIR=/gpfs/alpine/csc380/proj-shared/LBPM/cmake-3.21.0/bin
 #-I${MPICH_DIR}/include
 #-L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa
 
-HIPFLAGS = --amdgpu-target=gfx90a 
+#export HIPFLAGS="--amdgpu-target=gfx90a --save-temps"
+
+
+#THIS IS HOW TO CHECK FOR SPILLS (example)
+# hipcc -c -g -ggdb --save-temps Color.hip
 
 # configure
 rm -rf CMake*

tests/TestCommD3Q19.cpp

@@ -270,17 +270,19 @@ int main(int argc, char **argv)
 		//.......................................................................
 
 		//...........................................................................
-		comm.barrier();
+		//comm.barrier();
 		if (rank == 0) cout << "Domain set." << endl;
 		//...........................................................................
-
+                cout << flush;
 		//...........................................................................
 		if (rank==0)	printf ("Create ScaLBL_Communicator \n");
+                cout << flush;
 		// Create a communicator for the device (will use optimized layout)
 		ScaLBL_Communicator ScaLBL_Comm(Dm);
 		
 		int Npad=(Np/16 + 2)*16;
 		if (rank==0)    printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N);
+                cout << flush;
 		auto neighborList= new int[18*Npad];
 		IntArray Map(Nx,Ny,Nz);
 		Map.fill(-2);		
@@ -291,7 +293,8 @@ int main(int argc, char **argv)
 		//......................device distributions.................................
 		dist_mem_size = Np*sizeof(double);
 		if (rank==0)	printf ("Allocating distributions \n");
-		
+		cout << flush;
+
 		int *NeighborList;
 		int *dvcMap;
 		double *fq;
@@ -321,6 +324,9 @@ int main(int argc, char **argv)
 		ScaLBL_DeviceBarrier();
 		delete [] TmpMap;
 
+		if (rank==0)	printf("Map is copied to GPU \n");
+		cout << flush;
+
 		//...........................................................................
 
 		/*	// Write the communcation structure into a file for debugging
@@ -352,11 +358,13 @@ int main(int argc, char **argv)
 	fclose(CommFile);
 		 */
 		if (rank==0)	printf("Setting the distributions, size = : %i\n", Np);
+		cout << flush;
+		
 		//...........................................................................
 		GlobalFlipScaLBL_D3Q19_Init(fq_host, Map, Np, Nx-2, Ny-2, Nz-2, iproc,jproc,kproc,nprocx,nprocy,nprocz);
 		ScaLBL_CopyToDevice(fq, fq_host, 19*dist_mem_size);
 		ScaLBL_DeviceBarrier();
-		comm.barrier();
+		//comm.barrier();
 		//*************************************************************************
 		// First timestep
 		ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL
@@ -376,6 +384,7 @@ int main(int argc, char **argv)
 		int timestep = 0;
 		if (rank==0) printf("********************************************************\n");
 		if (rank==0)	printf("No. of timesteps for timing: %i \n", 100);
+		cout << flush;
 
 		//.......create and start timer............
 		double starttime,stoptime,cputime;
@@ -426,8 +435,11 @@ int main(int argc, char **argv)
 		// communication bandwidth includes both send and recieve
 		if (rank==0) printf("Communication bandwidth (per process)= %f Gbit/sec \n",ScaLBL_Comm.CommunicationCount*64*timestep/1e9);
 		if (rank==0) printf("Aggregated communication bandwidth = %f Gbit/sec \n",nprocs*ScaLBL_Comm.CommunicationCount*64*timestep/1e9);
+		cout << flush;
+
 	}
 	// ****************************************************
+	cout << fflush;
 	comm.barrier();
 	Utilities::shutdown();
 	// ****************************************************

tests/lbpm_color_simulator.cpp

@@ -23,7 +23,7 @@ int main( int argc, char **argv )
 {
 
 	// Initialize
-	Utilities::startup( argc, argv );
+        Utilities::startup( argc, argv, false );
 
 	{ // Limit scope so variables that contain communicators will free before MPI_Finialize
 
@@ -198,7 +198,7 @@ int main( int argc, char **argv )
 
 
 	} // Limit scope so variables that contain communicators will free before MPI_Finialize
-
+	cout << flush;
 	Utilities::shutdown();
 	return 0;
 }

tests/lbpm_permeability_simulator.cpp

@@ -24,7 +24,7 @@ using namespace std;
 int main(int argc, char **argv)
 {
 	// Initialize MPI
-    Utilities::startup( argc, argv );
+    Utilities::startup( argc, argv, false );
     Utilities::MPI comm( MPI_COMM_WORLD );
     int rank = comm.getRank();
     int nprocs = comm.getSize();
@@ -49,6 +49,7 @@ int main(int argc, char **argv)
 		MRT.Initialize();   // initializing the model will set initial conditions for variables
 		MRT.Run();	 
 		MRT.VelocityField();
+		cout << flush;
 	}
     Utilities::shutdown();
 }