OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Getting the CMake build working on titan

Browse Source
This commit is contained in:
Mark Berrill 2013-12-05 15:00:31 -05:00
parent 0430704252
commit d4742eb6c5
11 changed files with 273 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@ -22,7 +22,7 @@ CHECK_ENABLE_FLAG( ONLY_BUILD_DOCS 0 )
# Set testing paramaters
SET( DROP_METHOD "http" )
SET( DROP_SITE "oblivion.engr.colostate.edu" )
SET( DROP_LOCATION "/CDash/submit.php?project=AMR-MHD" )
SET( DROP_LOCATION "/CDash/submit.php?project=LBPM-WIA" )
SET( TRIGGER_SITE "" )
SET( DROP_SITE_CDASH TRUE )
ENABLE_TESTING()
@ -86,6 +86,7 @@ ENDIF()
# Configure external packages
IF ( NOT ONLY_BUILD_DOCS )
CONFIGURE_MPI() # MPI must be before other libraries
CONFIGURE_CUDA()
CONFIGURE_LBPM()
CONFIGURE_LINE_COVERAGE()
ENDIF()

45
README.titan
View File

@ -1,8 +1,49 @@
# INSTRUCTIONS FOR COMPILING AND RUNNING ON TITAN
# Note: there are 3 directories of interest:
# source directory - assumed to be located at ${HOME}/LBPM-WIA for these instructions, but may be anywhere
# build directory - current directory used for building, may not be the source directory, generally should be on lustre
# install directory - installation directory, optional, may be the same as the build directory,
# generally should be on lustre, specified with "-D LBPM_INSTALL_DIR:PATH="path_tot_install"
# load the module for cuda
module load cudatoolkit
# build executables
make all
# load the module for cmake
module load cmake
# configure
rm -rf CMake*
cmake \
-D CMAKE_C_COMPILER:PATH=cc \
-D CMAKE_CXX_COMPILER:PATH=CC \
-D CMAKE_CXX_COMPILER:PATH=CC \
-D CMAKE_C_FLAGS="-DCBUB" \
-D CMAKE_CXX_FLAGS="-DCBUB" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Debug \
-D CUDA_FLAGS="-arch sm_35" \
-D CUDA_HOST_COMPILER="/usr/bin/gcc" \
-D USE_CUDA=1 \
${HOME}/LBPM-WIA
# Build and install (may be done in parallel)
make install -j 8
# Run the tests (optional, requires that were are in an interactive session)
# Note: to run the tests and submit the results to CDash, see "run_ctest_titan"
# in sample_scripts, which uses the "config_titan" and "ctest_titan.ctest"
# Run the fast tests:
ctest -E WEEKLY
# Run all tests
ctest

40
cmake/libraries.cmake
View File

@ -7,6 +7,39 @@ MACRO ( CONFIGURE_LINE_COVERAGE )
ENDMACRO ()
# Macro to configure CUDA
MACRO ( CONFIGURE_CUDA )
SET(CUDA_FLAGS ${CUDA_NVCC_FLAGS})
SET( CUDA_FIND_QUIETLY 1)
IF(USE_CUDA)
INCLUDE ( FindCUDA )
IF ( NOT CUDA_FOUND )
MESSAGE ( FATAL_ERROR "CUDA not found" )
ENDIF()
SET(CUDA_NVCC_FLAGS ${CUDA_FLAGS})
IF(NOT CUDA_NVCC_FLAGS)
# Set minimum requirements
SET(CUDA_NVCC_FLAGS "-arch=sm_20")
ELSE()
STRING(REPLACE " " ";" CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS})
ENDIF()
ENDIF()
IF ( NOT USE_CUDA )
MESSAGE ( "Not using CUDA" )
ELSE ()
INCLUDE_DIRECTORIES ( ${CUDA_INCLUDE_DIRS} )
ADD_DEFINITIONS ( "-D USE_CUDA" )
MESSAGE ( "Using CUDA " ${CUDA_VERSION} )
MESSAGE ( " CUDA_LIBRARIES = " ${CUDA_LIBRARIES} )
MESSAGE ( " CUDA_INCLUDE = " ${CUDA_INCLUDE_DIRS} )
MESSAGE ( " CUDA_NVCC_FLAGS = " ${CUDA_NVCC_FLAGS} )
MESSAGE ( " CUDA_TOOLKIT_ROOT = " ${CUDA_TOOLKIT_ROOT_DIR} )
SET (EXTERNAL_LIBS ${EXTERNAL_LIBS} ${CUDA_LIBRARIES} )
ENDIF ()
ENDMACRO ()
# Macro to find and configure the MPI libraries
MACRO ( CONFIGURE_MPI )
# Determine if we want to use MPI
@ -44,6 +77,9 @@ MACRO ( CONFIGURE_MPI )
ENDIF()
ELSEIF ( MPI_COMPILER )
# The mpi compiler should take care of everything
IF ( NOT MPIEXEC )
MESSAGE( FATAL_ERROR "MPI_EXEC should be set" )
ENDIF()
ELSE()
# Perform the default search for MPI
INCLUDE ( FindMPI )
@ -53,6 +89,10 @@ MACRO ( CONFIGURE_MPI )
INCLUDE_DIRECTORIES ( ${MPI_INCLUDE_PATH} )
SET ( MPI_INCLUDE ${MPI_INCLUDE_PATH} )
ENDIF()
# Set defaults
IF ( NOT MPIEXEC_NUMPROC_FLAG )
SET(MPIEXEC_NUMPROC_FLAG "-n")
ENDIF()
# Check if we need to use MPI for serial tests
CHECK_ENABLE_FLAG( USE_MPI_FOR_SERIAL_TESTS 0 )
# Set the definitions

36
cmake/macros.cmake
View File

@ -1,9 +1,15 @@
INCLUDE(CheckCSourceCompiles)
# Macro to print all variables
MACRO( PRINT_ALL_VARIABLES )
GET_CMAKE_PROPERTY(_variableNames VARIABLES)
FOREACH(_variableName ${_variableNames})
message(STATUS "${_variableName}=${${_variableName}}")
ENDFOREACH()
ENDMACRO()
# Add a package to the LBPM library
MACRO( ADD_LBPM_LIBRARY PACKAGE )
#INCLUDE_DIRECTORIES ( ${LBPM_INSTALL_DIR}/include/${PACKAGE} )
@ -24,6 +30,7 @@ MACRO (BEGIN_PACKAGE_CONFIG PACKAGE)
SET( CSOURCES "" )
SET( FSOURCES "" )
SET( M4FSOURCES "" )
SET( CUSOURCES "" )
SET( SOURCES "" )
SET( CURPACKAGE ${PACKAGE} )
ENDMACRO ()
@ -40,11 +47,15 @@ MACRO (FIND_FILES)
# Find the C++ sources
SET( T_CXXSOURCES "" )
FILE( GLOB T_CXXSOURCES "*.cc" "*.cpp" "*.cxx" "*.C" )
# Find the C++ sources
SET( T_CXXSOURCES "" )
FILE( GLOB T_CXXSOURCES "*.cu" )
# Add all found files to the current lists
SET( HEADERS ${HEADERS} ${T_HEADERS} )
SET( CXXSOURCES ${CXXSOURCES} ${T_CXXSOURCES} )
SET( CSOURCES ${CSOURCES} ${T_CSOURCES} )
SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} )
SET( CUSOURCES ${SOURCES} ${T_CUSOURCES} )
ENDMACRO()
@ -59,11 +70,15 @@ MACRO (FIND_FILES_PATH IN_PATH)
# Find the C++ sources
SET( T_CXXSOURCES "" )
FILE( GLOB T_CXXSOURCES "${IN_PATH}/*.cc" "${IN_PATH}/*.cpp" "${IN_PATH}/*.cxx" "${IN_PATH}/*.C" )
# Find the CUDA sources
SET( T_CUSOURCES "" )
FILE( GLOB T_CUSOURCES "${IN_PATH}/*.cu" )
# Add all found files to the current lists
SET( HEADERS ${HEADERS} ${T_HEADERS} )
SET( CXXSOURCES ${CXXSOURCES} ${T_CXXSOURCES} )
SET( CSOURCES ${CSOURCES} ${T_CSOURCES} )
SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} )
SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_CUSOURCES} )
SET( CUSOURCES ${SOURCES} ${T_CUSOURCES} )
ENDMACRO()
@ -90,8 +105,10 @@ MACRO( INSTALL_LBPM_TARGET PACKAGE )
#CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/${HFILE} ${LBPM_INSTALL_DIR}/include/${CURPACKAGE}/${HFILE} COPYONLY )
CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/${HFILE} ${LBPM_INSTALL_DIR}/include/${HFILE} COPYONLY )
ENDFOREACH()
# Configure the CUDA files
CUDA_COMPILE( CUOBJS ${CUSOURCES} )
# Add the library
ADD_LIBRARY( ${PACKAGE} ${LIB_TYPE} ${SOURCES} )
ADD_LIBRARY( ${PACKAGE} ${LIB_TYPE} ${SOURCES} ${CUOBJS} )
SET( TEST_DEP_LIST ${PACKAGE} ${TEST_DEP_LIST} )
TARGET_LINK_LIBRARIES( ${PACKAGE} ${COVERAGE_LIBS} ${SYSTEM_LIBS} ${LDLIBS} )
TARGET_LINK_LIBRARIES( ${PACKAGE} ${LAPACK_LIBS} ${BLAS_LIBS} )
@ -204,12 +221,12 @@ MACRO( SET_COMPILE_FLAGS )
ENDIF()
ELSEIF ( ${CMAKE_BUILD_TYPE} STREQUAL "Release" )
IF ( USING_MICROSOFT )
SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} /O2" )
SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} /O2" )
SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} /O3" )
SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} /O3" )
SET(CONFIGURATION Release )
ELSE()
SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -O2" )
SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -O2" )
SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -O3" )
SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -O3" )
ENDIF()
ELSE()
MESSAGE(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
@ -233,6 +250,7 @@ MACRO( ADD_LBPM_EXE_DEP EXE )
# Add the libraries
TARGET_LINK_LIBRARIES( ${EXE} ${LBPM_LIBS} )
# Add external libraries
TARGET_LINK_LIBRARIES( ${EXE} ${EXTERNAL_LIBS} )
IF ( USE_MPI )
TARGET_LINK_LIBRARIES( ${EXE} ${MPI_LINK_FLAGS} ${MPI_LIBRARIES} )
ENDIF()

11
gpu/exe/CMakeLists.txt
View File

@ -1,6 +1,11 @@
#INSTALL_LBPM_EXE( lb1_MRT_mpi )
#INSTALL_LBPM_EXE( lb2_Color )
#INSTALL_LBPM_EXE( lb2_Color_mpi )
INSTALL_LBPM_EXE( lb2_Color )
INSTALL_LBPM_EXE( lb2_Color_mpi )
#INSTALL_LBPM_EXE( lb2_Color_pBC_wia_mpi )
INSTALL_LBPM_EXE( lb2_Color_wia_mpi )
# Run the serial ConstrainedBubble inputs as a weekly test
CONFIGURE_FILE( ${LBPM_SOURCE_DIR}/example/ConstrainedBubble/Color.in ${CMAKE_CURRENT_BINARY_DIR}/Color.in COPYONLY )
CONFIGURE_FILE( ${LBPM_SOURCE_DIR}/example/ConstrainedBubble/Domain.in ${CMAKE_CURRENT_BINARY_DIR}/Domain.in COPYONLY )
ADD_LBPM_WEEKLY_TEST( lb2_Color_wia_mpi 1 )

39
sample_scripts/config_build_titan Executable file
View File

@ -0,0 +1,39 @@
# Set the modules and enviornmental variables
source $MODULESHOME/init/bash
module load cudatoolkit
module load cmake
export MPICH_RDMA_ENABLED_CUDA=1
module swap cray-mpich2 cray-mpich2/5.6.3
# Remove CMake files from previous configures
rm -rf CMake*
# Configure
cmake \
-D CMAKE_C_COMPILER:PATH=cc \
-D CMAKE_CXX_COMPILER:PATH=CC \
-D CMAKE_CXX_COMPILER:PATH=CC \
-D CMAKE_C_FLAGS="-DCBUB" \
-D CMAKE_CXX_FLAGS="-DCBUB" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Debug \
-D CUDA_FLAGS="-arch sm_35" \
-D CUDA_HOST_COMPILER="/usr/bin/gcc" \
-D USE_CUDA=1 \
${HOME}/LBPM-WIA
# Build the code
make install -j 8
# Run the fast tests
ctest -E WEEKLY
# Run the slow tests
# ctest -R WEEKLY -VV

15
sample_scripts/config_titan Executable file
View File

@ -0,0 +1,15 @@
cmake \
-D CMAKE_C_COMPILER:PATH=cc \
-D CMAKE_CXX_COMPILER:PATH=CC \
-D CMAKE_CXX_COMPILER:PATH=CC \
-D CMAKE_C_FLAGS="-DCBUB" \
-D CMAKE_CXX_FLAGS="-DCBUB" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Debug \
-D CUDA_FLAGS="-arch sm_35" \
-D CUDA_HOST_COMPILER="/usr/bin/gcc" \
-D USE_CUDA=1 \
${HOME}/LBPM-WIA

51
sample_scripts/ctest_titan.ctest Executable file
View File

@ -0,0 +1,51 @@
# Sample script to configure, build, run, and submit the results
# Set some basic properties
SET( CTEST_CMAKE_COMMAND "./config_titan" )
SET( CTEST_DASHBOARD "Experimental" )
SET( CTEST_BUILD_NAME "all_tests" )
SET( CTEST_OPTIONS PARALLEL_LEVEL 1 )
# SET( CTEST_OPTIONS EXCLUDE WEEKLY PARALLEL_LEVEL 1 )
### Should not need to edit anything below this line
# Set the test properties
SITE_NAME( HOSTNAME )
SET( CTEST_PROJECT_NAME "LBPM-WIA" )
SET( CTEST_SOURCE_DIRECTORY "." )
SET( CTEST_BINARY_DIRECTORY "." )
SET( CTEST_TEST_TIMEOUT 300 )
SET( CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS 500 )
SET( CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS 500 )
SET( NIGHTLY_START_TIME "18:00:00 EST" )
SET( CTEST_NIGHTLY_START_TIME "22:00:00 EST" )
SET ( CTEST_COMMAND "\"${CTEST_EXECUTABLE_NAME}\" -D ${CTEST_DASHBOARD}" )
SET ( CTEST_CONFIGURE_COMMAND ${CTEST_CMAKE_COMMAND} )
SET ( CTEST_BUILD_COMMAND "make install -j 8" )
# Clear the binary directory and create an initial cache
CTEST_EMPTY_BINARY_DIRECTORY (${CTEST_BINARY_DIRECTORY})
FILE(WRITE "${CTEST_BINARY_DIRECTORY}/CMakeCache.txt" "CTEST_TEST_CTEST:BOOL=1")
# Configure and run the tests
SET( CTEST_SITE ${HOSTNAME} )
CTEST_START("${CTEST_DASHBOARD}")
CTEST_UPDATE()
CTEST_CONFIGURE()
CTEST_BUILD()
CTEST_TEST( ${CTEST_OPTIONS} )
# Submit the results to oblivion
SET ( CTEST_DROP_METHOD "http" )
SET ( CTEST_DROP_SITE "oblivion.engr.colostate.edu" )
SET ( CTEST_DROP_LOCATION "/CDash/submit.php?project=LBPM-WIA" )
SET ( CTEST_DROP_SITE_CDASH TRUE )
SET ( DROP_SITE_CDASH TRUE )
CTEST_SUBMIT()
# Clean up
exec_program("make distclean")

11
sample_scripts/run_ctest_titan Executable file
View File

@ -0,0 +1,11 @@
# Sample script to configure, build, run, and submit the results
# Set the modules and enviornmental variables
source $MODULESHOME/init/bash
module load cudatoolkit
module load cmake
export MPICH_RDMA_ENABLED_CUDA=1
module swap cray-mpich2 cray-mpich2/5.6.3
# Configure and run ctest
ctest -S ctest_titan.ctest -VV

13
tests/CMakeLists.txt
View File

@ -5,7 +5,12 @@ CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/cylindertest ${CMAKE_CURRENT_BINARY_
# Add the tests
ADD_LBPM_TEST( pmmc_cylinder )
ADD_LBPM_TEST( TestCylinderAreas )
ADD_LBPM_TEST( TestSphereCurvature )
ADD_LBPM_TEST( TestSphereCurvature )
# Sample test that will run with 1, 2, and 4 processors, failing with 4 or more procs
ADD_LBPM_TEST_1_2_4( hello_world )
SET_TESTS_PROPERTIES( hello_world PROPERTIES ENVIRONMENT "MPICH_RDMA_ENABLED_CUDA=0")
SET_TESTS_PROPERTIES( hello_world_2procs PROPERTIES ENVIRONMENT "MPICH_RDMA_ENABLED_CUDA=0")
SET_TESTS_PROPERTIES( hello_world_4procs PROPERTIES ENVIRONMENT "MPICH_RDMA_ENABLED_CUDA=0")

28
tests/hello_world.cpp Normal file
View File

@ -0,0 +1,28 @@
#include <iostream>
#include "mpi.h"
int main (int argc, char **argv)
{
int rank,nprocs;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
for (int i=0; i<nprocs; i++) {
if ( rank==i )
printf("%i of %i: Hello world\n",rank,nprocs);
MPI_Barrier(MPI_COMM_WORLD);
}
// set the error code
// Note: the error code should be consistent across all processors
int error = 0;
if ( nprocs==4 )
error = 1;
// Finished
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
return error;
}