From d4742eb6c5b9f5a514a6e51997f52c2879f2d7d7 Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Thu, 5 Dec 2013 15:00:31 -0500 Subject: [PATCH] Getting the CMake build working on titan --- CMakeLists.txt | 3 +- README.titan | 45 +++++++++++++++++++++++++-- cmake/libraries.cmake | 40 ++++++++++++++++++++++++ cmake/macros.cmake | 36 ++++++++++++++++------ gpu/exe/CMakeLists.txt | 11 +++++-- sample_scripts/config_build_titan | 39 +++++++++++++++++++++++ sample_scripts/config_titan | 15 +++++++++ sample_scripts/ctest_titan.ctest | 51 +++++++++++++++++++++++++++++++ sample_scripts/run_ctest_titan | 11 +++++++ tests/CMakeLists.txt | 13 +++++--- tests/hello_world.cpp | 28 +++++++++++++++++ 11 files changed, 273 insertions(+), 19 deletions(-) create mode 100755 sample_scripts/config_build_titan create mode 100755 sample_scripts/config_titan create mode 100755 sample_scripts/ctest_titan.ctest create mode 100755 sample_scripts/run_ctest_titan create mode 100644 tests/hello_world.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c9297816..f84f54aa 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,7 +22,7 @@ CHECK_ENABLE_FLAG( ONLY_BUILD_DOCS 0 ) # Set testing paramaters SET( DROP_METHOD "http" ) SET( DROP_SITE "oblivion.engr.colostate.edu" ) -SET( DROP_LOCATION "/CDash/submit.php?project=AMR-MHD" ) +SET( DROP_LOCATION "/CDash/submit.php?project=LBPM-WIA" ) SET( TRIGGER_SITE "" ) SET( DROP_SITE_CDASH TRUE ) ENABLE_TESTING() @@ -86,6 +86,7 @@ ENDIF() # Configure external packages IF ( NOT ONLY_BUILD_DOCS ) CONFIGURE_MPI() # MPI must be before other libraries + CONFIGURE_CUDA() CONFIGURE_LBPM() CONFIGURE_LINE_COVERAGE() ENDIF() diff --git a/README.titan b/README.titan index ae03d591..39615ea9 100644 --- a/README.titan +++ b/README.titan @@ -1,8 +1,49 @@ # INSTRUCTIONS FOR COMPILING AND RUNNING ON TITAN +# Note: there are 3 directories of interest: +# source directory - assumed to be located at ${HOME}/LBPM-WIA for these instructions, but may be anywhere +# build directory - current directory used for building, may not be the source directory, generally should be on lustre +# install directory - installation directory, optional, may be the same as the build directory, +# generally should be on lustre, specified with "-D LBPM_INSTALL_DIR:PATH="path_tot_install" + # load the module for cuda module load cudatoolkit -# build executables -make all + +# load the module for cmake +module load cmake + + +# configure +rm -rf CMake* +cmake \ + -D CMAKE_C_COMPILER:PATH=cc \ + -D CMAKE_CXX_COMPILER:PATH=CC \ + -D CMAKE_CXX_COMPILER:PATH=CC \ + -D CMAKE_C_FLAGS="-DCBUB" \ + -D CMAKE_CXX_FLAGS="-DCBUB" \ + -D MPI_COMPILER:BOOL=TRUE \ + -D MPIEXEC=aprun \ + -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ + -D CMAKE_BUILD_TYPE:STRING=Debug \ + -D CUDA_FLAGS="-arch sm_35" \ + -D CUDA_HOST_COMPILER="/usr/bin/gcc" \ + -D USE_CUDA=1 \ + ${HOME}/LBPM-WIA + + +# Build and install (may be done in parallel) +make install -j 8 + + +# Run the tests (optional, requires that were are in an interactive session) +# Note: to run the tests and submit the results to CDash, see "run_ctest_titan" +# in sample_scripts, which uses the "config_titan" and "ctest_titan.ctest" +# Run the fast tests: +ctest -E WEEKLY +# Run all tests +ctest + + + diff --git a/cmake/libraries.cmake b/cmake/libraries.cmake index de2a3423..9a76419f 100644 --- a/cmake/libraries.cmake +++ b/cmake/libraries.cmake @@ -7,6 +7,39 @@ MACRO ( CONFIGURE_LINE_COVERAGE ) ENDMACRO () +# Macro to configure CUDA +MACRO ( CONFIGURE_CUDA ) + SET(CUDA_FLAGS ${CUDA_NVCC_FLAGS}) + SET( CUDA_FIND_QUIETLY 1) + IF(USE_CUDA) + INCLUDE ( FindCUDA ) + IF ( NOT CUDA_FOUND ) + MESSAGE ( FATAL_ERROR "CUDA not found" ) + ENDIF() + SET(CUDA_NVCC_FLAGS ${CUDA_FLAGS}) + IF(NOT CUDA_NVCC_FLAGS) + # Set minimum requirements + SET(CUDA_NVCC_FLAGS "-arch=sm_20") + ELSE() + STRING(REPLACE " " ";" CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS}) + ENDIF() + ENDIF() + + IF ( NOT USE_CUDA ) + MESSAGE ( "Not using CUDA" ) + ELSE () + INCLUDE_DIRECTORIES ( ${CUDA_INCLUDE_DIRS} ) + ADD_DEFINITIONS ( "-D USE_CUDA" ) + MESSAGE ( "Using CUDA " ${CUDA_VERSION} ) + MESSAGE ( " CUDA_LIBRARIES = " ${CUDA_LIBRARIES} ) + MESSAGE ( " CUDA_INCLUDE = " ${CUDA_INCLUDE_DIRS} ) + MESSAGE ( " CUDA_NVCC_FLAGS = " ${CUDA_NVCC_FLAGS} ) + MESSAGE ( " CUDA_TOOLKIT_ROOT = " ${CUDA_TOOLKIT_ROOT_DIR} ) + SET (EXTERNAL_LIBS ${EXTERNAL_LIBS} ${CUDA_LIBRARIES} ) + ENDIF () +ENDMACRO () + + # Macro to find and configure the MPI libraries MACRO ( CONFIGURE_MPI ) # Determine if we want to use MPI @@ -44,6 +77,9 @@ MACRO ( CONFIGURE_MPI ) ENDIF() ELSEIF ( MPI_COMPILER ) # The mpi compiler should take care of everything + IF ( NOT MPIEXEC ) + MESSAGE( FATAL_ERROR "MPI_EXEC should be set" ) + ENDIF() ELSE() # Perform the default search for MPI INCLUDE ( FindMPI ) @@ -53,6 +89,10 @@ MACRO ( CONFIGURE_MPI ) INCLUDE_DIRECTORIES ( ${MPI_INCLUDE_PATH} ) SET ( MPI_INCLUDE ${MPI_INCLUDE_PATH} ) ENDIF() + # Set defaults + IF ( NOT MPIEXEC_NUMPROC_FLAG ) + SET(MPIEXEC_NUMPROC_FLAG "-n") + ENDIF() # Check if we need to use MPI for serial tests CHECK_ENABLE_FLAG( USE_MPI_FOR_SERIAL_TESTS 0 ) # Set the definitions diff --git a/cmake/macros.cmake b/cmake/macros.cmake index a70fe00f..e9785bf9 100644 --- a/cmake/macros.cmake +++ b/cmake/macros.cmake @@ -1,9 +1,15 @@ - - - INCLUDE(CheckCSourceCompiles) +# Macro to print all variables +MACRO( PRINT_ALL_VARIABLES ) + GET_CMAKE_PROPERTY(_variableNames VARIABLES) + FOREACH(_variableName ${_variableNames}) + message(STATUS "${_variableName}=${${_variableName}}") + ENDFOREACH() +ENDMACRO() + + # Add a package to the LBPM library MACRO( ADD_LBPM_LIBRARY PACKAGE ) #INCLUDE_DIRECTORIES ( ${LBPM_INSTALL_DIR}/include/${PACKAGE} ) @@ -24,6 +30,7 @@ MACRO (BEGIN_PACKAGE_CONFIG PACKAGE) SET( CSOURCES "" ) SET( FSOURCES "" ) SET( M4FSOURCES "" ) + SET( CUSOURCES "" ) SET( SOURCES "" ) SET( CURPACKAGE ${PACKAGE} ) ENDMACRO () @@ -40,11 +47,15 @@ MACRO (FIND_FILES) # Find the C++ sources SET( T_CXXSOURCES "" ) FILE( GLOB T_CXXSOURCES "*.cc" "*.cpp" "*.cxx" "*.C" ) + # Find the C++ sources + SET( T_CXXSOURCES "" ) + FILE( GLOB T_CXXSOURCES "*.cu" ) # Add all found files to the current lists SET( HEADERS ${HEADERS} ${T_HEADERS} ) SET( CXXSOURCES ${CXXSOURCES} ${T_CXXSOURCES} ) SET( CSOURCES ${CSOURCES} ${T_CSOURCES} ) SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ) + SET( CUSOURCES ${SOURCES} ${T_CUSOURCES} ) ENDMACRO() @@ -59,11 +70,15 @@ MACRO (FIND_FILES_PATH IN_PATH) # Find the C++ sources SET( T_CXXSOURCES "" ) FILE( GLOB T_CXXSOURCES "${IN_PATH}/*.cc" "${IN_PATH}/*.cpp" "${IN_PATH}/*.cxx" "${IN_PATH}/*.C" ) + # Find the CUDA sources + SET( T_CUSOURCES "" ) + FILE( GLOB T_CUSOURCES "${IN_PATH}/*.cu" ) # Add all found files to the current lists SET( HEADERS ${HEADERS} ${T_HEADERS} ) SET( CXXSOURCES ${CXXSOURCES} ${T_CXXSOURCES} ) SET( CSOURCES ${CSOURCES} ${T_CSOURCES} ) - SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ) + SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_CUSOURCES} ) + SET( CUSOURCES ${SOURCES} ${T_CUSOURCES} ) ENDMACRO() @@ -90,8 +105,10 @@ MACRO( INSTALL_LBPM_TARGET PACKAGE ) #CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/${HFILE} ${LBPM_INSTALL_DIR}/include/${CURPACKAGE}/${HFILE} COPYONLY ) CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/${HFILE} ${LBPM_INSTALL_DIR}/include/${HFILE} COPYONLY ) ENDFOREACH() + # Configure the CUDA files + CUDA_COMPILE( CUOBJS ${CUSOURCES} ) # Add the library - ADD_LIBRARY( ${PACKAGE} ${LIB_TYPE} ${SOURCES} ) + ADD_LIBRARY( ${PACKAGE} ${LIB_TYPE} ${SOURCES} ${CUOBJS} ) SET( TEST_DEP_LIST ${PACKAGE} ${TEST_DEP_LIST} ) TARGET_LINK_LIBRARIES( ${PACKAGE} ${COVERAGE_LIBS} ${SYSTEM_LIBS} ${LDLIBS} ) TARGET_LINK_LIBRARIES( ${PACKAGE} ${LAPACK_LIBS} ${BLAS_LIBS} ) @@ -204,12 +221,12 @@ MACRO( SET_COMPILE_FLAGS ) ENDIF() ELSEIF ( ${CMAKE_BUILD_TYPE} STREQUAL "Release" ) IF ( USING_MICROSOFT ) - SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} /O2" ) - SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} /O2" ) + SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} /O3" ) + SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} /O3" ) SET(CONFIGURATION Release ) ELSE() - SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -O2" ) - SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -O2" ) + SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -O3" ) + SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -O3" ) ENDIF() ELSE() MESSAGE(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}") @@ -233,6 +250,7 @@ MACRO( ADD_LBPM_EXE_DEP EXE ) # Add the libraries TARGET_LINK_LIBRARIES( ${EXE} ${LBPM_LIBS} ) # Add external libraries + TARGET_LINK_LIBRARIES( ${EXE} ${EXTERNAL_LIBS} ) IF ( USE_MPI ) TARGET_LINK_LIBRARIES( ${EXE} ${MPI_LINK_FLAGS} ${MPI_LIBRARIES} ) ENDIF() diff --git a/gpu/exe/CMakeLists.txt b/gpu/exe/CMakeLists.txt index 1fedd882..3cdb087d 100755 --- a/gpu/exe/CMakeLists.txt +++ b/gpu/exe/CMakeLists.txt @@ -1,6 +1,11 @@ #INSTALL_LBPM_EXE( lb1_MRT_mpi ) -#INSTALL_LBPM_EXE( lb2_Color ) -#INSTALL_LBPM_EXE( lb2_Color_mpi ) +INSTALL_LBPM_EXE( lb2_Color ) +INSTALL_LBPM_EXE( lb2_Color_mpi ) #INSTALL_LBPM_EXE( lb2_Color_pBC_wia_mpi ) INSTALL_LBPM_EXE( lb2_Color_wia_mpi ) - + + +# Run the serial ConstrainedBubble inputs as a weekly test +CONFIGURE_FILE( ${LBPM_SOURCE_DIR}/example/ConstrainedBubble/Color.in ${CMAKE_CURRENT_BINARY_DIR}/Color.in COPYONLY ) +CONFIGURE_FILE( ${LBPM_SOURCE_DIR}/example/ConstrainedBubble/Domain.in ${CMAKE_CURRENT_BINARY_DIR}/Domain.in COPYONLY ) +ADD_LBPM_WEEKLY_TEST( lb2_Color_wia_mpi 1 ) diff --git a/sample_scripts/config_build_titan b/sample_scripts/config_build_titan new file mode 100755 index 00000000..a796fe45 --- /dev/null +++ b/sample_scripts/config_build_titan @@ -0,0 +1,39 @@ +# Set the modules and enviornmental variables +source $MODULESHOME/init/bash +module load cudatoolkit +module load cmake +export MPICH_RDMA_ENABLED_CUDA=1 +module swap cray-mpich2 cray-mpich2/5.6.3 + + +# Remove CMake files from previous configures +rm -rf CMake* + + +# Configure +cmake \ + -D CMAKE_C_COMPILER:PATH=cc \ + -D CMAKE_CXX_COMPILER:PATH=CC \ + -D CMAKE_CXX_COMPILER:PATH=CC \ + -D CMAKE_C_FLAGS="-DCBUB" \ + -D CMAKE_CXX_FLAGS="-DCBUB" \ + -D MPI_COMPILER:BOOL=TRUE \ + -D MPIEXEC=aprun \ + -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ + -D CMAKE_BUILD_TYPE:STRING=Debug \ + -D CUDA_FLAGS="-arch sm_35" \ + -D CUDA_HOST_COMPILER="/usr/bin/gcc" \ + -D USE_CUDA=1 \ + ${HOME}/LBPM-WIA + + +# Build the code +make install -j 8 + + +# Run the fast tests +ctest -E WEEKLY + + +# Run the slow tests +# ctest -R WEEKLY -VV diff --git a/sample_scripts/config_titan b/sample_scripts/config_titan new file mode 100755 index 00000000..f74f9442 --- /dev/null +++ b/sample_scripts/config_titan @@ -0,0 +1,15 @@ +cmake \ + -D CMAKE_C_COMPILER:PATH=cc \ + -D CMAKE_CXX_COMPILER:PATH=CC \ + -D CMAKE_CXX_COMPILER:PATH=CC \ + -D CMAKE_C_FLAGS="-DCBUB" \ + -D CMAKE_CXX_FLAGS="-DCBUB" \ + -D MPI_COMPILER:BOOL=TRUE \ + -D MPIEXEC=aprun \ + -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ + -D CMAKE_BUILD_TYPE:STRING=Debug \ + -D CUDA_FLAGS="-arch sm_35" \ + -D CUDA_HOST_COMPILER="/usr/bin/gcc" \ + -D USE_CUDA=1 \ + ${HOME}/LBPM-WIA + diff --git a/sample_scripts/ctest_titan.ctest b/sample_scripts/ctest_titan.ctest new file mode 100755 index 00000000..9b12d011 --- /dev/null +++ b/sample_scripts/ctest_titan.ctest @@ -0,0 +1,51 @@ +# Sample script to configure, build, run, and submit the results + + +# Set some basic properties +SET( CTEST_CMAKE_COMMAND "./config_titan" ) +SET( CTEST_DASHBOARD "Experimental" ) +SET( CTEST_BUILD_NAME "all_tests" ) +SET( CTEST_OPTIONS PARALLEL_LEVEL 1 ) +# SET( CTEST_OPTIONS EXCLUDE WEEKLY PARALLEL_LEVEL 1 ) + + +### Should not need to edit anything below this line + +# Set the test properties +SITE_NAME( HOSTNAME ) +SET( CTEST_PROJECT_NAME "LBPM-WIA" ) +SET( CTEST_SOURCE_DIRECTORY "." ) +SET( CTEST_BINARY_DIRECTORY "." ) +SET( CTEST_TEST_TIMEOUT 300 ) +SET( CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS 500 ) +SET( CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS 500 ) +SET( NIGHTLY_START_TIME "18:00:00 EST" ) +SET( CTEST_NIGHTLY_START_TIME "22:00:00 EST" ) + +SET ( CTEST_COMMAND "\"${CTEST_EXECUTABLE_NAME}\" -D ${CTEST_DASHBOARD}" ) +SET ( CTEST_CONFIGURE_COMMAND ${CTEST_CMAKE_COMMAND} ) +SET ( CTEST_BUILD_COMMAND "make install -j 8" ) + +# Clear the binary directory and create an initial cache +CTEST_EMPTY_BINARY_DIRECTORY (${CTEST_BINARY_DIRECTORY}) +FILE(WRITE "${CTEST_BINARY_DIRECTORY}/CMakeCache.txt" "CTEST_TEST_CTEST:BOOL=1") + +# Configure and run the tests +SET( CTEST_SITE ${HOSTNAME} ) +CTEST_START("${CTEST_DASHBOARD}") +CTEST_UPDATE() +CTEST_CONFIGURE() +CTEST_BUILD() +CTEST_TEST( ${CTEST_OPTIONS} ) + +# Submit the results to oblivion +SET ( CTEST_DROP_METHOD "http" ) +SET ( CTEST_DROP_SITE "oblivion.engr.colostate.edu" ) +SET ( CTEST_DROP_LOCATION "/CDash/submit.php?project=LBPM-WIA" ) +SET ( CTEST_DROP_SITE_CDASH TRUE ) +SET ( DROP_SITE_CDASH TRUE ) +CTEST_SUBMIT() + +# Clean up +exec_program("make distclean") + diff --git a/sample_scripts/run_ctest_titan b/sample_scripts/run_ctest_titan new file mode 100755 index 00000000..b041cad5 --- /dev/null +++ b/sample_scripts/run_ctest_titan @@ -0,0 +1,11 @@ +# Sample script to configure, build, run, and submit the results + +# Set the modules and enviornmental variables +source $MODULESHOME/init/bash +module load cudatoolkit +module load cmake +export MPICH_RDMA_ENABLED_CUDA=1 +module swap cray-mpich2 cray-mpich2/5.6.3 + +# Configure and run ctest +ctest -S ctest_titan.ctest -VV diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a6e1d1ac..cbe253ce 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,7 +5,12 @@ CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/cylindertest ${CMAKE_CURRENT_BINARY_ # Add the tests ADD_LBPM_TEST( pmmc_cylinder ) ADD_LBPM_TEST( TestCylinderAreas ) -ADD_LBPM_TEST( TestSphereCurvature ) - - - +ADD_LBPM_TEST( TestSphereCurvature ) + +# Sample test that will run with 1, 2, and 4 processors, failing with 4 or more procs +ADD_LBPM_TEST_1_2_4( hello_world ) +SET_TESTS_PROPERTIES( hello_world PROPERTIES ENVIRONMENT "MPICH_RDMA_ENABLED_CUDA=0") +SET_TESTS_PROPERTIES( hello_world_2procs PROPERTIES ENVIRONMENT "MPICH_RDMA_ENABLED_CUDA=0") +SET_TESTS_PROPERTIES( hello_world_4procs PROPERTIES ENVIRONMENT "MPICH_RDMA_ENABLED_CUDA=0") + + diff --git a/tests/hello_world.cpp b/tests/hello_world.cpp new file mode 100644 index 00000000..db47c647 --- /dev/null +++ b/tests/hello_world.cpp @@ -0,0 +1,28 @@ +#include +#include "mpi.h" + + +int main (int argc, char **argv) +{ + int rank,nprocs; + MPI_Init(&argc,&argv); + MPI_Comm_rank(MPI_COMM_WORLD,&rank); + MPI_Comm_size(MPI_COMM_WORLD,&nprocs); + + for (int i=0; i