Updating build to support HIP as a language

CMakeLists.txt

@@ -125,21 +125,7 @@ IF ( USE_CUDA )
     ADD_DEFINITIONS( -DUSE_CUDA )
     ENABLE_LANGUAGE( CUDA )
 ELSEIF ( USE_HIP )
-    IF ( NOT DEFINED HIP_PATH )
-        IF ( NOT DEFINED ENV{HIP_PATH} )
-            SET( HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed" )
-        ELSE()
-            SET( HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to which HIP has been installed" )
-        ENDIF()
-    ENDIF()
-    SET( CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH} )
-    FIND_PACKAGE( HIP REQUIRED )
-    FIND_PACKAGE( CUDA QUIET )
-    MESSAGE( "HIP Found")
-    MESSAGE( "   HIP version:      ${HIP_VERSION_STRING}")
-    MESSAGE( "   HIP platform:     ${HIP_PLATFORM}")
-    MESSAGE( "   HIP Include Path: ${HIP_INCLUDE_DIRS}")
-    MESSAGE( "   HIP Libraries:    ${HIP_LIBRARIES}")
+    ENABLE_LANGUAGE( HIP )
     ADD_DEFINITIONS( -DUSE_HIP )
 ENDIF()
 
@@ -180,8 +166,7 @@ IF ( NOT ONLY_BUILD_DOCS )
     IF ( USE_CUDA )
         ADD_PACKAGE_SUBDIRECTORY( cuda )
     ELSEIF ( USE_HIP )
-        ADD_SUBDIRECTORY( hip )
-        SET( LBPM_LIBRARIES lbpm-hip lbpm-wia )
+        ADD_PACKAGE_SUBDIRECTORY( hip )
     ELSE()
         ADD_PACKAGE_SUBDIRECTORY( cpu )
     ENDIF()

cmake/macros.cmake

@@ -193,6 +193,9 @@ MACRO( FIND_FILES )
     # Find the CUDA sources
     SET( T_CUDASOURCES "" )
     FILE( GLOB T_CUDASOURCES "*.cu" )
+    # Find the HIP sources
+    SET( T_HIPSOURCES "" )
+    FILE( GLOB T_HIPSOURCES "*.hip" )
     # Find the C sources
     SET( T_CSOURCES "" )
     FILE( GLOB T_CSOURCES "*.c" )
@@ -212,10 +215,11 @@ MACRO( FIND_FILES )
     SET( HEADERS ${HEADERS} ${T_HEADERS} )
     SET( CXXSOURCES ${CXXSOURCES} ${T_CXXSOURCES} )
     SET( CUDASOURCES ${CUDASOURCES} ${T_CUDASOURCES} )
+    SET( HIPSOURCES ${HIPSOURCES} ${T_HIPSOURCES} )
     SET( CSOURCES ${CSOURCES} ${T_CSOURCES} )
     SET( FSOURCES ${FSOURCES} ${T_FSOURCES} )
     SET( M4FSOURCES ${M4FSOURCES} ${T_M4FSOURCES} )
-    SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${T_M4FSOURCES} ${CUDASOURCES} )
+    SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${T_M4FSOURCES} ${CUDASOURCES} ${HIPSOURCES} )
 ENDMACRO()
 
 
@@ -227,6 +231,9 @@ MACRO( FIND_FILES_PATH IN_PATH )
     # Find the CUDA sources
     SET( T_CUDASOURCES "" )
     FILE( GLOB T_CUDASOURCES "${IN_PATH}/*.cu" )
+    # Find the HIP sources
+    SET( T_HIPSOURCES "" )
+    FILE( GLOB T_HIPSOURCES "${IN_PATH}/*.hip" )
     # Find the C sources
     SET( T_CSOURCES "" )
     FILE( GLOB T_CSOURCES "${IN_PATH}/*.c" )
@@ -246,9 +253,10 @@ MACRO( FIND_FILES_PATH IN_PATH )
     SET( HEADERS ${HEADERS} ${T_HEADERS} )
     SET( CXXSOURCES ${CXXSOURCES} ${T_CXXSOURCES} )
     SET( CUDASOURCES ${CUDASOURCES} ${T_CUDASOURCES} )
+    SET( HIPSOURCES ${HIPSOURCES} ${T_HIPSOURCES} )
     SET( CSOURCES ${CSOURCES} ${T_CSOURCES} )
     SET( FSOURCES ${FSOURCES} ${T_FSOURCES} )
-    SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${CUDASOURCES} )
+    SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${CUDASOURCES} ${HIPSOURCES} )
 ENDMACRO()
 
 

hip/CMakeLists.txt

@@ -1,10 +0,0 @@
-SET( HIP_SEPERABLE_COMPILATION ON )
-FILE( GLOB HIP_SOURCES "*.cu" )
-SET_SOURCE_FILES_PROPERTIES( ${HIP_SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1 )
-HIP_ADD_LIBRARY( lbpm-hip ${HIP_SOURCES} SHARED HIPCC_OPTIONS ${HIP_HIPCC_OPTIONS} HCC_OPTIONS ${HIP_HCC_OPTIONS} NVCC_OPTIONS ${HIP_NVCC_OPTIONS} ${HIP_NVCC_FLAGS} )
-#TARGET_LINK_LIBRARIES( lbpm-hip /opt/rocm-3.3.0/lib/libhip_hcc.so )
-#TARGET_LINK_LIBRARIES( lbpm-wia lbpm-hip )
-#ADD_DEPENDENCIES( lbpm-hip copy-include )
-
-
-

sample_scripts/configure_spock_hip

@@ -1,40 +1,37 @@
-module load cmake
-#module load llvm-amdgpu
-#module load hip
-#module load llvm
-#module load rocm/4.2.0
-
-module load craype-accel-amd-gfx908
-module load PrgEnv-cray
+module load PrgEnv-gnu
 module load rocm/4.2.0
+module load cray-mpich
+module load cray-hdf5-parallel
+#module load craype-accel-amd-gfx908
 
-export TPL_DIR=/ccs/proj/csc380/mcclurej/spock/install
 
-#export PE_MPICH_GTL_DIR_amd_gfx908="-L/opt/cray/pe/mpich/8.1.4/gtl/lib"
-#export PE_MPICH_GTL_LIBS_amd_gfx908="-lmpi_gtl_hsa"
+## These must be set before compiling so the executable picks up GTL
+export PE_MPICH_GTL_DIR_amd_gfx908="-L/opt/cray/pe/mpich/8.1.4/gtl/lib"
+export PE_MPICH_GTL_LIBS_amd_gfx908="-lmpi_gtl_hsa"
+
+
+# Need a new version of cmake
+export CMAKE_DIR=/gpfs/alpine/csc380/proj-shared/LBPM/cmake-3.21.0/bin
+
 
 # configure
 rm -rf CMake*
-cmake                                           \
+${CMAKE_DIR}/cmake                              \
     -D CMAKE_BUILD_TYPE:STRING=Release          \
-    -D CMAKE_C_COMPILER:PATH=cc                 \
     -D CMAKE_CXX_COMPILER:PATH=CC               \
     -D CMAKE_CXX_STANDARD=14                    \
-    -D CMAKE_CXX_FLAGS="-I${ROCM_PATH}/include -L${ROCM_PATH}/lib -lamdhip64 -lhsa-runtime64 -I${MPICH_DIR}/include -L${MPICH_DIR}/lib -lmpi -L/opt/cray/pe/mpich/8.1.4/gtl/lib -lmpi_gtl_hsa" \
+    -D DISABLE_GOLD:BOOL=TRUE                   \
+    -D DISABLE_LTO:BOOL=TRUE                    \
     -D USE_HIP=1                                \
-        -D LINK_LIBRARIES=${HIP_PATH}/lib/libamdhip64.so \
-    -D USE_CUDA=0                               \
-        -D CMAKE_CUDA_FLAGS="-arch sm_70 -Xptxas=-v -Xptxas -dlcm=cg -lineinfo" \
-        -D CMAKE_CUDA_HOST_COMPILER="gcc"       \
+        -D CMAKE_HIP_COMPILER_TOOLKIT_ROOT=$ROCM_PATH/hip \
     -D USE_MPI=1                                \
     -D MPI_SKIP_SEARCH=1                        \
     -D MPIEXEC="srun"                           \
     -D USE_HDF5=1                               \
-        -D HDF5_DIRECTORY="${TPL_DIR}/hdf5"     \
-     -D USE_SILO=0                              \
-        -D SILO_DIRECTORY="${TPL_DIR}/silo"     \
-    -D USE_DOXYGEN:BOOL=false		        \
+        -D HDF5_DIRECTORY="${HDF5_DIR}"         \
+    -D USE_SILO=0                               \
     -D USE_TIMER=0			        \
+    -D USE_DOXYGEN:BOOL=false		        \
     ~/LBPM-WIA
 
 

tests/CMakeLists.txt

@@ -55,7 +55,7 @@ ADD_LBPM_TEST( TestSubphase )
 ADD_LBPM_TEST( TestTorus )
 ADD_LBPM_TEST( TestTorusEvolve )
 ADD_LBPM_TEST( TestTopo3D )
-ADD_LBPM_TEST( TestFluxBC )
+ADD_LBPM_TEST( TestFluxBC.cpp )
 ADD_LBPM_TEST( TestMap )
 #ADD_LBPM_TEST( TestMRT )
 #ADD_LBPM_TEST( TestColorGrad )