From a55a030c3c479bf1b90da4234037181500a7b8e6 Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Mon, 2 Aug 2021 13:54:30 -0400 Subject: [PATCH] Updating build to support HIP as a language --- CMakeLists.txt | 19 +-------- cmake/macros.cmake | 12 +++++- hip/{BGK.cu => BGK.hip} | 0 hip/CMakeLists.txt | 10 ----- hip/{Color.cu => Color.hip} | 0 hip/{CudaExtras.cu => CudaExtras.hip} | 0 hip/{D3Q19.cu => D3Q19.hip} | 0 hip/{D3Q7.cu => D3Q7.hip} | 0 hip/{D3Q7BC.cu => D3Q7BC.hip} | 0 hip/{Extras.cu => Extras.hip} | 0 hip/{FreeLee.cu => FreeLee.hip} | 0 hip/{Greyscale.cu => Greyscale.hip} | 0 hip/{GreyscaleColor.cu => GreyscaleColor.hip} | 0 hip/{Ion.cu => Ion.hip} | 0 hip/{MRT.cu => MRT.hip} | 0 hip/{MixedGradient.cu => MixedGradient.hip} | 0 hip/{Poisson.cu => Poisson.hip} | 0 hip/{Stokes.cu => Stokes.hip} | 0 hip/{dfh.cu => dfh.hip} | 0 sample_scripts/configure_spock_hip | 41 +++++++++---------- tests/CMakeLists.txt | 2 +- 21 files changed, 32 insertions(+), 52 deletions(-) rename hip/{BGK.cu => BGK.hip} (100%) delete mode 100644 hip/CMakeLists.txt rename hip/{Color.cu => Color.hip} (100%) rename hip/{CudaExtras.cu => CudaExtras.hip} (100%) rename hip/{D3Q19.cu => D3Q19.hip} (100%) rename hip/{D3Q7.cu => D3Q7.hip} (100%) rename hip/{D3Q7BC.cu => D3Q7BC.hip} (100%) rename hip/{Extras.cu => Extras.hip} (100%) rename hip/{FreeLee.cu => FreeLee.hip} (100%) rename hip/{Greyscale.cu => Greyscale.hip} (100%) rename hip/{GreyscaleColor.cu => GreyscaleColor.hip} (100%) rename hip/{Ion.cu => Ion.hip} (100%) rename hip/{MRT.cu => MRT.hip} (100%) rename hip/{MixedGradient.cu => MixedGradient.hip} (100%) rename hip/{Poisson.cu => Poisson.hip} (100%) rename hip/{Stokes.cu => Stokes.hip} (100%) rename hip/{dfh.cu => dfh.hip} (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index c9b7e66a..b60f741f 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -125,21 +125,7 @@ IF ( USE_CUDA ) ADD_DEFINITIONS( -DUSE_CUDA ) ENABLE_LANGUAGE( CUDA ) ELSEIF ( USE_HIP ) - IF ( NOT DEFINED HIP_PATH ) - IF ( NOT DEFINED ENV{HIP_PATH} ) - SET( HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed" ) - ELSE() - SET( HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to which HIP has been installed" ) - ENDIF() - ENDIF() - SET( CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH} ) - FIND_PACKAGE( HIP REQUIRED ) - FIND_PACKAGE( CUDA QUIET ) - MESSAGE( "HIP Found") - MESSAGE( " HIP version: ${HIP_VERSION_STRING}") - MESSAGE( " HIP platform: ${HIP_PLATFORM}") - MESSAGE( " HIP Include Path: ${HIP_INCLUDE_DIRS}") - MESSAGE( " HIP Libraries: ${HIP_LIBRARIES}") + ENABLE_LANGUAGE( HIP ) ADD_DEFINITIONS( -DUSE_HIP ) ENDIF() @@ -180,8 +166,7 @@ IF ( NOT ONLY_BUILD_DOCS ) IF ( USE_CUDA ) ADD_PACKAGE_SUBDIRECTORY( cuda ) ELSEIF ( USE_HIP ) - ADD_SUBDIRECTORY( hip ) - SET( LBPM_LIBRARIES lbpm-hip lbpm-wia ) + ADD_PACKAGE_SUBDIRECTORY( hip ) ELSE() ADD_PACKAGE_SUBDIRECTORY( cpu ) ENDIF() diff --git a/cmake/macros.cmake b/cmake/macros.cmake index 47150cf0..56e9a684 100644 --- a/cmake/macros.cmake +++ b/cmake/macros.cmake @@ -193,6 +193,9 @@ MACRO( FIND_FILES ) # Find the CUDA sources SET( T_CUDASOURCES "" ) FILE( GLOB T_CUDASOURCES "*.cu" ) + # Find the HIP sources + SET( T_HIPSOURCES "" ) + FILE( GLOB T_HIPSOURCES "*.hip" ) # Find the C sources SET( T_CSOURCES "" ) FILE( GLOB T_CSOURCES "*.c" ) @@ -212,10 +215,11 @@ MACRO( FIND_FILES ) SET( HEADERS ${HEADERS} ${T_HEADERS} ) SET( CXXSOURCES ${CXXSOURCES} ${T_CXXSOURCES} ) SET( CUDASOURCES ${CUDASOURCES} ${T_CUDASOURCES} ) + SET( HIPSOURCES ${HIPSOURCES} ${T_HIPSOURCES} ) SET( CSOURCES ${CSOURCES} ${T_CSOURCES} ) SET( FSOURCES ${FSOURCES} ${T_FSOURCES} ) SET( M4FSOURCES ${M4FSOURCES} ${T_M4FSOURCES} ) - SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${T_M4FSOURCES} ${CUDASOURCES} ) + SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${T_M4FSOURCES} ${CUDASOURCES} ${HIPSOURCES} ) ENDMACRO() @@ -227,6 +231,9 @@ MACRO( FIND_FILES_PATH IN_PATH ) # Find the CUDA sources SET( T_CUDASOURCES "" ) FILE( GLOB T_CUDASOURCES "${IN_PATH}/*.cu" ) + # Find the HIP sources + SET( T_HIPSOURCES "" ) + FILE( GLOB T_HIPSOURCES "${IN_PATH}/*.hip" ) # Find the C sources SET( T_CSOURCES "" ) FILE( GLOB T_CSOURCES "${IN_PATH}/*.c" ) @@ -246,9 +253,10 @@ MACRO( FIND_FILES_PATH IN_PATH ) SET( HEADERS ${HEADERS} ${T_HEADERS} ) SET( CXXSOURCES ${CXXSOURCES} ${T_CXXSOURCES} ) SET( CUDASOURCES ${CUDASOURCES} ${T_CUDASOURCES} ) + SET( HIPSOURCES ${HIPSOURCES} ${T_HIPSOURCES} ) SET( CSOURCES ${CSOURCES} ${T_CSOURCES} ) SET( FSOURCES ${FSOURCES} ${T_FSOURCES} ) - SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${CUDASOURCES} ) + SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${CUDASOURCES} ${HIPSOURCES} ) ENDMACRO() diff --git a/hip/BGK.cu b/hip/BGK.hip similarity index 100% rename from hip/BGK.cu rename to hip/BGK.hip diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt deleted file mode 100644 index 9e613960..00000000 --- a/hip/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -SET( HIP_SEPERABLE_COMPILATION ON ) -FILE( GLOB HIP_SOURCES "*.cu" ) -SET_SOURCE_FILES_PROPERTIES( ${HIP_SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1 ) -HIP_ADD_LIBRARY( lbpm-hip ${HIP_SOURCES} SHARED HIPCC_OPTIONS ${HIP_HIPCC_OPTIONS} HCC_OPTIONS ${HIP_HCC_OPTIONS} NVCC_OPTIONS ${HIP_NVCC_OPTIONS} ${HIP_NVCC_FLAGS} ) -#TARGET_LINK_LIBRARIES( lbpm-hip /opt/rocm-3.3.0/lib/libhip_hcc.so ) -#TARGET_LINK_LIBRARIES( lbpm-wia lbpm-hip ) -#ADD_DEPENDENCIES( lbpm-hip copy-include ) - - - diff --git a/hip/Color.cu b/hip/Color.hip similarity index 100% rename from hip/Color.cu rename to hip/Color.hip diff --git a/hip/CudaExtras.cu b/hip/CudaExtras.hip similarity index 100% rename from hip/CudaExtras.cu rename to hip/CudaExtras.hip diff --git a/hip/D3Q19.cu b/hip/D3Q19.hip similarity index 100% rename from hip/D3Q19.cu rename to hip/D3Q19.hip diff --git a/hip/D3Q7.cu b/hip/D3Q7.hip similarity index 100% rename from hip/D3Q7.cu rename to hip/D3Q7.hip diff --git a/hip/D3Q7BC.cu b/hip/D3Q7BC.hip similarity index 100% rename from hip/D3Q7BC.cu rename to hip/D3Q7BC.hip diff --git a/hip/Extras.cu b/hip/Extras.hip similarity index 100% rename from hip/Extras.cu rename to hip/Extras.hip diff --git a/hip/FreeLee.cu b/hip/FreeLee.hip similarity index 100% rename from hip/FreeLee.cu rename to hip/FreeLee.hip diff --git a/hip/Greyscale.cu b/hip/Greyscale.hip similarity index 100% rename from hip/Greyscale.cu rename to hip/Greyscale.hip diff --git a/hip/GreyscaleColor.cu b/hip/GreyscaleColor.hip similarity index 100% rename from hip/GreyscaleColor.cu rename to hip/GreyscaleColor.hip diff --git a/hip/Ion.cu b/hip/Ion.hip similarity index 100% rename from hip/Ion.cu rename to hip/Ion.hip diff --git a/hip/MRT.cu b/hip/MRT.hip similarity index 100% rename from hip/MRT.cu rename to hip/MRT.hip diff --git a/hip/MixedGradient.cu b/hip/MixedGradient.hip similarity index 100% rename from hip/MixedGradient.cu rename to hip/MixedGradient.hip diff --git a/hip/Poisson.cu b/hip/Poisson.hip similarity index 100% rename from hip/Poisson.cu rename to hip/Poisson.hip diff --git a/hip/Stokes.cu b/hip/Stokes.hip similarity index 100% rename from hip/Stokes.cu rename to hip/Stokes.hip diff --git a/hip/dfh.cu b/hip/dfh.hip similarity index 100% rename from hip/dfh.cu rename to hip/dfh.hip diff --git a/sample_scripts/configure_spock_hip b/sample_scripts/configure_spock_hip index dc6461f4..dd02f163 100755 --- a/sample_scripts/configure_spock_hip +++ b/sample_scripts/configure_spock_hip @@ -1,40 +1,37 @@ -module load cmake -#module load llvm-amdgpu -#module load hip -#module load llvm -#module load rocm/4.2.0 - -module load craype-accel-amd-gfx908 -module load PrgEnv-cray +module load PrgEnv-gnu module load rocm/4.2.0 +module load cray-mpich +module load cray-hdf5-parallel +#module load craype-accel-amd-gfx908 -export TPL_DIR=/ccs/proj/csc380/mcclurej/spock/install -#export PE_MPICH_GTL_DIR_amd_gfx908="-L/opt/cray/pe/mpich/8.1.4/gtl/lib" -#export PE_MPICH_GTL_LIBS_amd_gfx908="-lmpi_gtl_hsa" +## These must be set before compiling so the executable picks up GTL +export PE_MPICH_GTL_DIR_amd_gfx908="-L/opt/cray/pe/mpich/8.1.4/gtl/lib" +export PE_MPICH_GTL_LIBS_amd_gfx908="-lmpi_gtl_hsa" + + +# Need a new version of cmake +export CMAKE_DIR=/gpfs/alpine/csc380/proj-shared/LBPM/cmake-3.21.0/bin + # configure rm -rf CMake* -cmake \ +${CMAKE_DIR}/cmake \ -D CMAKE_BUILD_TYPE:STRING=Release \ - -D CMAKE_C_COMPILER:PATH=cc \ -D CMAKE_CXX_COMPILER:PATH=CC \ -D CMAKE_CXX_STANDARD=14 \ - -D CMAKE_CXX_FLAGS="-I${ROCM_PATH}/include -L${ROCM_PATH}/lib -lamdhip64 -lhsa-runtime64 -I${MPICH_DIR}/include -L${MPICH_DIR}/lib -lmpi -L/opt/cray/pe/mpich/8.1.4/gtl/lib -lmpi_gtl_hsa" \ + -D DISABLE_GOLD:BOOL=TRUE \ + -D DISABLE_LTO:BOOL=TRUE \ -D USE_HIP=1 \ - -D LINK_LIBRARIES=${HIP_PATH}/lib/libamdhip64.so \ - -D USE_CUDA=0 \ - -D CMAKE_CUDA_FLAGS="-arch sm_70 -Xptxas=-v -Xptxas -dlcm=cg -lineinfo" \ - -D CMAKE_CUDA_HOST_COMPILER="gcc" \ + -D CMAKE_HIP_COMPILER_TOOLKIT_ROOT=$ROCM_PATH/hip \ -D USE_MPI=1 \ -D MPI_SKIP_SEARCH=1 \ -D MPIEXEC="srun" \ -D USE_HDF5=1 \ - -D HDF5_DIRECTORY="${TPL_DIR}/hdf5" \ - -D USE_SILO=0 \ - -D SILO_DIRECTORY="${TPL_DIR}/silo" \ - -D USE_DOXYGEN:BOOL=false \ + -D HDF5_DIRECTORY="${HDF5_DIR}" \ + -D USE_SILO=0 \ -D USE_TIMER=0 \ + -D USE_DOXYGEN:BOOL=false \ ~/LBPM-WIA diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 320de0a6..1fce4c1a 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -55,7 +55,7 @@ ADD_LBPM_TEST( TestSubphase ) ADD_LBPM_TEST( TestTorus ) ADD_LBPM_TEST( TestTorusEvolve ) ADD_LBPM_TEST( TestTopo3D ) -ADD_LBPM_TEST( TestFluxBC ) +ADD_LBPM_TEST( TestFluxBC.cpp ) ADD_LBPM_TEST( TestMap ) #ADD_LBPM_TEST( TestMRT ) #ADD_LBPM_TEST( TestColorGrad )