From 3beb380bf31cf0cc67ad73ecf58cee36d8f97d85 Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Fri, 26 Mar 2021 13:20:49 -0400 Subject: [PATCH 1/4] Minor update to MPI class --- common/MPI.I | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/common/MPI.I b/common/MPI.I index 0361e108..6d44858a 100644 --- a/common/MPI.I +++ b/common/MPI.I @@ -595,7 +595,15 @@ template void MPI_CLASS::sendrecv( const TYPE *sendbuf, int sendcount, int dest, int sendtag, TYPE *recvbuf, int recvcount, int source, int recvtag ) const { - ERROR( "Not implimented" ); + if ( getSize() == 1 ) { + ASSERT( dest == 0 ); + ASSERT( source == 0 ); + ASSERT( sendcount == recvcount ); + ASSERT( sendtag == recvtag ); + memcpy( recvbuf, sendbuf, sendcount * sizeof( TYPE ) ); + } else { + ERROR( "Not implimented for " + std::string( typeid( TYPE ).name() ) ); + } } #else template From 8f8d23632a2fec8714dd3a076d2308553dcc2427 Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Wed, 12 May 2021 13:19:50 -0400 Subject: [PATCH 2/4] Changes for spock --- CMakeLists.txt | 1 + cmake/macros.cmake | 336 +++++++++++++++++++--------- hip/FreeLee.cu | 18 +- sample_scripts/configure_spock_TPLs | 33 +++ sample_scripts/configure_spock_hip | 30 +++ 5 files changed, 299 insertions(+), 119 deletions(-) create mode 100755 sample_scripts/configure_spock_TPLs create mode 100755 sample_scripts/configure_spock_hip diff --git a/CMakeLists.txt b/CMakeLists.txt index 8f500927..a45f71f5 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,6 @@ # Set some CMake properties CMAKE_MINIMUM_REQUIRED( VERSION 3.9 ) +CMAKE_POLICY( SET CMP0115 OLD ) MESSAGE("====================") diff --git a/cmake/macros.cmake b/cmake/macros.cmake index 8030dfa4..c8da0a45 100644 --- a/cmake/macros.cmake +++ b/cmake/macros.cmake @@ -1,3 +1,7 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 3.18.3) +CMAKE_POLICY( SET CMP0057 NEW ) +CMAKE_POLICY( SET CMP0115 OLD ) + INCLUDE(CheckCCompilerFlag) INCLUDE(CheckCSourceCompiles) INCLUDE(CheckCXXCompilerFlag) @@ -47,12 +51,24 @@ ELSEIF( NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION ) ENDIF() +# Check for gold linker +IF ( UNIX AND NOT APPLE AND NOT DISABLE_GOLD ) + EXECUTE_PROCESS( COMMAND ${CMAKE_CXX_COMPILER} -fuse-ld=gold -Wl,--version ERROR_QUIET OUTPUT_VARIABLE LD_VERSION ) + IF ( "${LD_VERSION}" MATCHES "GNU gold" ) + MESSAGE( "Using gold linker" ) + SET( GOLD_LINK_FLAGS "-fuse-ld=gold -Wl,--disable-new-dtags -Wl,--no-map-whole-files" ) + SET( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GOLD_LINK_FLAGS}" ) + SET( CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${GOLD_LINK_FLAGS}" ) + ENDIF() +ENDIF() + + # Add some default targets if they do not exist IF ( NOT TARGET copy-${PROJ}-Data ) ADD_CUSTOM_TARGET( copy-${PROJ}-Data ALL ) ENDIF() IF ( NOT TARGET copy-${PROJ}-include ) - ADD_CUSTOM_TARGET ( copy-${PROJ}-include ALL ) + ADD_CUSTOM_TARGET( copy-${PROJ}-include ALL ) ENDIF() @@ -75,7 +91,7 @@ ENDMACRO() MACRO( PRINT_ALL_VARIABLES ) GET_CMAKE_PROPERTY(_variableNames VARIABLES) FOREACH(_variableName ${_variableNames}) - message(STATUS "${_variableName}=${${_variableName}}") + MESSAGE(STATUS "${_variableName}=${${_variableName}}") ENDFOREACH() ENDMACRO() @@ -108,28 +124,28 @@ MACRO( CONVERT_M4_FORTRAN IN LOCAL_PATH OUT_PATH ) IF ( NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/${OUT_PATH}" ) FILE(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${OUT_PATH}" ) ENDIF() - CONFIGURE_FILE ( ${IN} ${IN} COPYONLY ) + CONFIGURE_FILE( ${IN} ${IN} COPYONLY ) IF ("${CMAKE_GENERATOR}" STREQUAL "Xcode") STRING(REGEX REPLACE ".F" ".o" OUT2 "${OUT}" ) STRING(REGEX REPLACE ";" " " COMPILE_CMD "${CMAKE_Fortran_COMPILER} -c ${OUT} ${CMAKE_Fortran_FLAGS} -o ${OUT2}") STRING(REGEX REPLACE "\\\\" "" COMPILE_CMD "${COMPILE_CMD}") MESSAGE("COMPILE_CMD = ${COMPILE_CMD}") SET( COMPILE_CMD ${COMPILE_CMD} ) - add_custom_command( + ADD_CUSTOM_COMMAND( OUTPUT ${OUT2} COMMAND m4 -I${LOCAL_PATH} -I${SAMRAI_FORTDIR} ${M4DIRS} ${IN} > ${OUT} COMMAND ${COMPILE_CMD} DEPENDS ${IN} ) - set_source_files_properties(${OUT2} PROPERTIES GENERATED true) + SET_SOURCE_FILES_PROPERTIES(${OUT2} PROPERTIES GENERATED true) SET( SOURCES ${SOURCES} "${OUT2}" ) ELSE() - add_custom_command( + ADD_CUSTOM_COMMAND( OUTPUT ${OUT} COMMAND m4 -I${LOCAL_PATH} -I${SAMRAI_FORTDIR} ${M4DIRS} ${M4_OPTIONS} ${IN} > ${OUT} DEPENDS ${IN} ) - set_source_files_properties(${OUT} PROPERTIES GENERATED true) + SET_SOURCE_FILES_PROPERTIES(${OUT} PROPERTIES GENERATED true) SET( SOURCES ${SOURCES} "${OUT}" ) ENDIF() ENDMACRO() @@ -145,6 +161,10 @@ ENDMACRO() # Add a project executable MACRO( ADD_${PROJ}_EXECUTABLE EXEFILE ) + IF ( TARGET ${EXEFILE} AND NOT INCLUDE_${EXEFILE} ) + MESSAGE( FATAL_ERROR "ADD_${PROJ}_EXECUTABLE should be called before adding tests" ) + ENDIF() + SET( INCLUDE_${EXEFILE} TRUE ) ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} ) INSTALL( TARGETS ${EXEFILE} DESTINATION ${${PROJ}_INSTALL_DIR}/bin ) ENDMACRO() @@ -164,7 +184,7 @@ ENDMACRO () # Find the source files -MACRO (FIND_FILES) +MACRO( FIND_FILES ) # Find the C/C++ headers SET( T_HEADERS "" ) FILE( GLOB T_HEADERS "*.h" "*.H" "*.hh" "*.hpp" "*.I" ) @@ -193,7 +213,7 @@ MACRO (FIND_FILES) SET( CSOURCES ${CSOURCES} ${T_CSOURCES} ) SET( FSOURCES ${FSOURCES} ${T_FSOURCES} ) SET( M4FSOURCES ${M4FSOURCES} ${T_M4FSOURCES} ) - SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${T_M4FSOURCES} ) + SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${T_M4FSOURCES} ${CUDASOURCES} ) ENDMACRO() @@ -303,7 +323,7 @@ MACRO( INSTALL_${PROJ}_TARGET PACKAGE ) IF ( TARGET write_repo_version ) ADD_DEPENDENCIES( ${PACKAGE} write_repo_version ) ENDIF() - ADD_DEPENDENCIES ( ${PACKAGE} copy-${PROJ}-include ) + ADD_DEPENDENCIES( ${PACKAGE} copy-${PROJ}-include ) IF ( NOT ${PROJ}_LIB ) INSTALL( TARGETS ${PACKAGE} DESTINATION ${${PROJ}_INSTALL_DIR}/lib ) ENDIF() @@ -340,7 +360,7 @@ ENDMACRO() # Macro to verify that a path has been set MACRO( VERIFY_PATH PATH_NAME ) IF ("${PATH_NAME}" STREQUAL "") - MESSAGE ( FATAL_ERROR "Path is not set: ${PATH_NAME}" ) + MESSAGE( FATAL_ERROR "Path is not set: ${PATH_NAME}" ) ENDIF() IF ( NOT EXISTS "${PATH_NAME}" ) MESSAGE( FATAL_ERROR "Path does not exist: ${PATH_NAME}" ) @@ -420,6 +440,9 @@ MACRO( IDENTIFY_COMPILER ) ELSEIF ( ${CMAKE_Fortran_COMPILER_ID} MATCHES "PGI") SET(USING_PGF90 TRUE) MESSAGE("Using pgf90") + ELSEIF( (${CMAKE_Fortran_COMPILER_ID} MATCHES "CRAY") OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "Cray") ) + SET(USING_CRAY TRUE) + MESSAGE("Using Cray") ELSEIF ( (${CMAKE_Fortran_COMPILER_ID} MATCHES "CLANG") OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "Clang") OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "FLANG") OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "Flang") ) SET(USING_FLANG TRUE) @@ -440,7 +463,7 @@ MACRO( SET_WARNINGS ) # Note: adding -Wlogical-op causes a wierd linking error on Titan using the nvcc wrapper: # /usr/bin/ld: cannot find gical-op: No such file or directory SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Woverloaded-virtual -Wsign-compare") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -Woverloaded-virtual -Wsign-compare -Wformat-security") ELSEIF ( USING_MSVC ) # Add Microsoft specifc compiler options SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /D _SCL_SECURE_NO_WARNINGS /D _CRT_SECURE_NO_WARNINGS /D _ITERATOR_DEBUG_LEVEL=0 /wd4267" ) @@ -448,7 +471,7 @@ MACRO( SET_WARNINGS ) ELSEIF ( USING_ICC ) # Add Intel specifc compiler options SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -Wall" ) - SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall" ) + SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall -wd1011" ) ELSEIF ( USING_CRAY ) # Add default compiler options SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS}") @@ -461,8 +484,8 @@ MACRO( SET_WARNINGS ) SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} --diag_suppress 111,128,185") ELSEIF ( USING_CLANG ) # Add default compiler options - SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -Wall") - SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall -Wno-missing-braces -Wmissing-field-initializers -ftemplate-depth=1024") + SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -Wall -Wextra") + SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic -Wno-missing-braces -Wmissing-field-initializers -ftemplate-depth=1024") ELSEIF ( USING_XL ) # Add default compiler options SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -Wall") @@ -651,7 +674,7 @@ MACRO( COPY_MESH_FILE MESHNAME ) ENDFOREACH() # We have either found the mesh or failed IF ( NOT MESHPATH ) - MESSAGE ( WARNING "Cannot find mesh: " ${MESHNAME} ) + MESSAGE( WARNING "Cannot find mesh: " ${MESHNAME} ) ELSE () SET( MESHPATH2 ) FOREACH( tmp ${MESHPATH} ) @@ -666,25 +689,16 @@ ENDMACRO() # Link the libraries to the given target MACRO( TARGET_LINK_EXTERNAL_LIBRARIES TARGET_NAME ) - FOREACH ( tmp ${TPL_LIBS} ) - TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} ) - ENDFOREACH() + # Include external libraries FOREACH ( tmp ${EXTERNAL_LIBS} ) TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} ) ENDFOREACH() - FOREACH ( tmp ${LAPACK_LIBS} ) + # Include libraries found through the TPL builder + FOREACH ( tmp ${TPL_LIBS} ) TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} ) ENDFOREACH() - FOREACH ( tmp ${BLAS_LIBS} ) - TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} ) - ENDFOREACH() - FOREACH ( tmp ${BLAS_LAPACK_LIBS} ) - TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} ) - ENDFOREACH() - FOREACH ( tmp ${MPI_LIBRARIES} ) - TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} ) - ENDFOREACH() - FOREACH ( tmp ${CMAKE_C_IMPLICIT_LINK_LIBRARIES} + # Include CMake implicit libraries + FOREACH ( tmp ${CMAKE_CUDA_IMPLICIT_LINK_LIBRARIES} ${CMAKE_C_IMPLICIT_LINK_LIBRARIES} ${CMAKE_CXX_IMPLICIT_LINK_LIBRARIES} ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES} ) TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} ) ENDFOREACH() @@ -715,8 +729,8 @@ ENDFUNCTION() MACRO( ADD_PROJ_EXE_DEP EXE ) # Add the package dependencies IF( ${PROJ}_TEST_LIB_EXISTS ) - ADD_DEPENDENCIES ( ${EXE} ${PACKAGE_TEST_LIB} ) - TARGET_LINK_LIBRARIES ( ${EXE} ${PACKAGE_TEST_LIB} ) + ADD_DEPENDENCIES( ${EXE} ${PACKAGE_TEST_LIB} ) + TARGET_LINK_LIBRARIES( ${EXE} ${PACKAGE_TEST_LIB} ) ENDIF() # Add the executable to the dependencies of check and build-test ADD_DEPENDENCIES( check ${EXE} ) @@ -783,7 +797,7 @@ FUNCTION( ADD_PROJ_PROVISIONAL_TEST EXEFILE ) SET( CXXFILE ${EXEFILE} ) SET( TESTS_SO_FAR ${TESTS_SO_FAR} ${EXEFILE} ) # Check if we want to add the test to all - IF ( NOT EXCLUDE_TESTS_FROM_ALL ) + IF ( NOT EXCLUDE_TESTS_FROM_ALL OR INCLUDE_${EXEFILE} ) ADD_EXECUTABLE( ${EXEFILE} ${CXXFILE} ) ELSE() ADD_EXECUTABLE( ${EXEFILE} EXCLUDE_FROM_ALL ${CXXFILE} ) @@ -818,123 +832,218 @@ MACRO( CREATE_TEST_NAME TEST ${ARGN} ) FOREACH( tmp ${ARGN}) SET( TESTNAME "${TESTNAME}--${tmp}") endforeach() - # STRING(REGEX REPLACE "--" "-" TESTNAME ${TESTNAME} ) SET( LAST_TESTNAME ${TESTNAME} PARENT_SCOPE ) ENDMACRO() # Function to add the resource locks to an executable -FUNCTION( ADD_RESOURCE_LOCK TESTNAME EXEFILE ${ARGN} ) +FUNCTION( ADD_RESOURCE TESTNAME RESOURCE ) + SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES RESOURCE_LOCK ${RESOURCE} ) +ENDFUNCTION() +FUNCTION( ADD_RESOURCE_LOCKS TESTNAME EXEFILE ${ARGN} ) IF ( NOT DISABLE_RESOURCE_LOCK ) IF ( NOT ARGN ) - SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES RESOURCE_LOCK ${EXEFILE} ) + ADD_RESOURCE( ${TESTNAME} ${EXEFILE} ) ELSE() FOREACH( tmp ${ARGN} ) - SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES RESOURCE_LOCK ${tmp} ) + ADD_RESOURCE( ${TESTNAME} ${tmp} ) ENDFOREACH() ENDIF() ENDIF() ENDFUNCTION() -# Add a executable as a test -FUNCTION( ADD_${PROJ}_TEST EXEFILE ${ARGN} ) +# Function to add a test +FUNCTION( CALL_ADD_TEST EXEFILE PROCS THREADS WEEKLY TESTBUILDER ${ARGN} ) # Check if we actually want to add the test KEEP_TEST( RESULT ) IF ( NOT RESULT ) + MESSAGE( "Discarding test: ${EXEFILE}" ) RETURN() ENDIF() - # Add the provisional test - ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} ) - CREATE_TEST_NAME( ${EXEFILE} ${ARGN} ) - IF ( USE_MPI_FOR_SERIAL_TESTS ) - ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" 1 $ ${ARGN} ) - SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none ) + # Check if we are dealing with a TestBuilder test + IF ( ${TESTBUILDER} ) + # Add the test to the TestBuilder sources + SET( TESTBUILDER_SOURCES ${TESTBUILDER_SOURCES} ${EXEFILE} PARENT_SCOPE ) ELSE() - ADD_TEST( NAME ${TESTNAME} COMMAND $ ${ARGN} ) + # Add the provisional test + ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} ) ENDIF() - SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS 1 ) - ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} ) -ENDFUNCTION() - - -# Add a executable as a weekly test -FUNCTION( ADD_${PROJ}_WEEKLY_TEST EXEFILE PROCS ${ARGN} ) - # Check if we actually want to add the test - KEEP_TEST( RESULT ) - IF ( NOT RESULT ) - RETURN() + # Create the test name + SET( TESTNAME ${EXEFILE} ) + #IF ( ${TESTBUILDER} ) + # SET( TESTNAME TB_${TESTNAME} ) + #ENDIF() + IF ( "${PROCS}" GREATER "1") + SET( TESTNAME ${TESTNAME}_${PROCS}procs ) ENDIF() - # Add the provisional test - ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} ) - IF( ${PROCS} STREQUAL "1" ) - CREATE_TEST_NAME( "${EXEFILE}_WEEKLY" ${ARGN} ) - ELSEIF( (USE_MPI OR USE_EXT_MPI) AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) ) - CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_WEEKLY" ${ARGN} ) + IF ( "${THREADS}" GREATER "1") + SET( TESTNAME ${TESTNAME}_${THREADS}threads ) ENDIF() - IF ( ${PROCS} GREATER ${TEST_MAX_PROCS} ) + IF ( ${WEEKLY} ) + SET( TESTNAME ${TESTNAME}_WEEKLY ) + ENDIF() + CREATE_TEST_NAME( ${TESTNAME} ${ARGN} ) + # Create the tests for ctest + MATH( EXPR TOT_PROCS "${PROCS} * ${THREADS}" ) + IF ( ${TOT_PROCS} EQUAL 0 ) + # Skip test (provisional) + ELSEIF ( ${TOT_PROCS} GREATER ${TEST_MAX_PROCS} ) MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})") - ELSEIF( ${PROCS} STREQUAL "1" ) - CREATE_TEST_NAME( "${EXEFILE}_WEEKLY" ${ARGN} ) - IF ( USE_MPI_FOR_SERIAL_TESTS ) - ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" 1 $ ${ARGN} ) - SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none ) + ELSEIF ( ( ${PROCS} STREQUAL "1" ) AND NOT USE_MPI_FOR_SERIAL_TESTS ) + IF ( ${TESTBUILDER} ) + ADD_TEST( NAME ${TESTNAME} COMMAND $ ${EXEFILE} ${ARGN} ) ELSE() ADD_TEST( NAME ${TESTNAME} COMMAND $ ${ARGN} ) ENDIF() - SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS 1 ) - ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} ) - ELSEIF( (USE_MPI OR USE_EXT_MPI) AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) ) - CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_WEEKLY" ${ARGN} ) - ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $ ${ARGN} ) - SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} ) + SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} ) + ADD_RESOURCE_LOCKS( ${TESTNAME} ${EXEFILE} ${ARGN} ) + ELSEIF ( USE_MPI OR USE_EXT_MPI ) + IF ( ${TESTBUILDER} ) + ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $ ${EXEFILE} ${ARGN} ) + ELSE() + ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $ ${ARGN} ) + ENDIF() SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none ) - ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} ) + SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} ) + ADD_RESOURCE_LOCKS( ${TESTNAME} ${EXEFILE} ${ARGN} ) ENDIF() ENDFUNCTION() -# Add a executable as a parallel test -FUNCTION( ADD_${PROJ}_TEST_PARALLEL EXEFILE PROCS ${ARGN} ) +# Functions to create a test builder +FUNCTION( INITIALIZE_TESTBUILDER ) + SET( TESTBUILDER_SOURCES PARENT_SCOPE ) # Check if we actually want to add the test KEEP_TEST( RESULT ) IF ( NOT RESULT ) RETURN() ENDIF() - # Add the provisional test - ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} ) - CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs" ${ARGN} ) - IF ( NOT ( USE_MPI OR USE_EXT_MPI ) ) - MESSAGE("Disabling test ${TESTNAME} (configured without MPI)") - ELSEIF ( ${PROCS} GREATER ${TEST_MAX_PROCS} ) - MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})") - ELSE() - ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $ ${ARGN} ) - SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none ) - SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} ) - ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} ) + # Check if the target does not exist (may not be added yet or we are re-configuring) + STRING(REGEX REPLACE "${${PROJ}_SOURCE_DIR}/" "" TB_TARGET "TB-${PROJ}-${CMAKE_CURRENT_SOURCE_DIR}" ) + STRING(REGEX REPLACE "/" "-" TB_TARGET ${TB_TARGET} ) + IF ( NOT TARGET ${TB_TARGET} ) + GLOBAL_SET( ${TB_TARGET}-BINDIR ) ENDIF() + # Check if test has already been added + IF ( NOT ${TB_TARGET}-BINDIR ) + GLOBAL_SET( ${TB_TARGET}-BINDIR "${CMAKE_CURRENT_BINARY_DIR}" ) + # Create the initial file + IF ( NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" ) + FILE( WRITE "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" "// Auto generated file\n\n" ) + ENDIF() + # The target has not been added + SET( CXXFILE "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" ) + SET( TESTS_SO_FAR ${TESTS_SO_FAR} ${TB_TARGET} ) + # Check if we want to add the test to all + IF ( NOT EXCLUDE_TESTS_FROM_ALL ) + ADD_EXECUTABLE( ${TB_TARGET} ${CXXFILE} ) + ELSE() + ADD_EXECUTABLE( ${TB_TARGET} EXCLUDE_FROM_ALL ${CXXFILE} ) + ENDIF() + SET_TARGET_PROPERTIES( ${TB_TARGET} PROPERTIES OUTPUT_NAME "TB" ) + ELSEIF ( NOT ${TB-BINDIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR} ) + # We are trying to add 2 different tests with the same name + MESSAGE( "Existing test: ${TB-BINDIR}/${TB_TARGET}" ) + MESSAGE( "New test: ${CMAKE_CURRENT_BINARY_DIR}/${TB_TARGET}}" ) + MESSAGE( FATAL_ERROR "Trying to add 2 different tests with the same name" ) + ENDIF() + SET( TB_TARGET ${TB_TARGET} PARENT_SCOPE ) +ENDFUNCTION() +FUNCTION( FINALIZE_TESTBUILDER ) + # Check if we actually want to add the test + KEEP_TEST( RESULT ) + IF ( NOT RESULT ) + RETURN() + ENDIF() + # Create the library for the test builder + IF ( TESTBUILDER_SOURCES ) + LIST( REMOVE_DUPLICATES TESTBUILDER_SOURCES ) + SET( TB_TARGET_SOURCES "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" ) + FOREACH ( tmp ${TESTBUILDER_SOURCES} ) + GET_FILENAME_COMPONENT( tmp ${tmp} ABSOLUTE ) + SET( TB_TARGET_SOURCES ${TB_TARGET_SOURCES} ${tmp} ) + ENDFOREACH() + TARGET_SOURCES( ${TB_TARGET} PUBLIC ${TB_TARGET_SOURCES} ) + ADD_DEPENDENCIES( ${TB_TARGET} copy-${PROJ}-include ) + ENDIF() + # Add the dependencies to the test builder + ADD_PROJ_EXE_DEP( ${TB_TARGET} ) + IF ( CURRENT_LIBRARY ) + IF ( NOT TARGET ${CURRENT_LIBRARY}-test ) + ADD_CUSTOM_TARGET( ${CURRENT_LIBRARY}-test ) + ENDIF() + ADD_DEPENDENCIES( ${CURRENT_LIBRARY}-test ${TB_TARGET} ) + ENDIF() + SET( TB_TARGET ${TB_TARGET} PARENT_SCOPE ) + # Finish generating TestBuilder.cpp + SET( TB_FILE "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder-tmp.cpp" ) + FILE( WRITE "${TB_FILE}" "// Auto generated file\n\n" ) + FILE( APPEND "${TB_FILE}" "#include \n" ) + FILE( APPEND "${TB_FILE}" "#include \n\n" ) + FOREACH( tmp ${TESTBUILDER_SOURCES} ) + FILE( APPEND "${TB_FILE}" "extern int ${tmp}( int argc, char *argv[] );\n" ) + ENDFOREACH() + FILE( APPEND "${TB_FILE}" "\n\n// Main Program\n" ) + FILE( APPEND "${TB_FILE}" "int main( int argc, char *argv[] )\n" ) + FILE( APPEND "${TB_FILE}" "{\n" ) + FILE( APPEND "${TB_FILE}" " if ( argc < 2 ) {\n" ) + FILE( APPEND "${TB_FILE}" " std::cerr << \"Invaild number of arguments for TestBuilder\";\n " ) + FILE( APPEND "${TB_FILE}" " return 1;\n" ) + FILE( APPEND "${TB_FILE}" " }\n\n" ) + FILE( APPEND "${TB_FILE}" " int rtn = 0;\n" ) + FILE( APPEND "${TB_FILE}" " if ( strcmp( argv[1], \"NULL\" ) == 0 ) {\n" ) + FOREACH( tmp ${TESTBUILDER_SOURCES} ) + FILE( APPEND "${TB_FILE}" " } else if ( strcmp( argv[1], \"${tmp}\" ) == 0 ) {\n" ) + FILE( APPEND "${TB_FILE}" " rtn = ${tmp}( argc-1, &argv[1] );\n" ) + ENDFOREACH() + FILE( APPEND "${TB_FILE}" " } else {\n" ) + FILE( APPEND "${TB_FILE}" " std::cerr << \"Unknown test\";\n" ) + FILE( APPEND "${TB_FILE}" " return 1;\n" ) + FILE( APPEND "${TB_FILE}" " }\n\n" ) + FILE( APPEND "${TB_FILE}" " return rtn;\n" ) + FILE( APPEND "${TB_FILE}" "}\n" ) + EXECUTE_PROCESS( COMMAND ${CMAKE_COMMAND} -E copy_if_different "${TB_FILE}" "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" ) ENDFUNCTION() -# Add a parallel test that may use both MPI and threads -# This allows us to correctly compute the number of processors used by the test +# Convience functions to add a test +MACRO( ADD_${PROJ}_TEST EXEFILE ${ARGN} ) + CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE FALSE ${ARGN} ) +ENDMACRO() +MACRO( ADD_${PROJ}_TEST_PARALLEL EXEFILE PROCS ${ARGN} ) + CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 FALSE FALSE ${ARGN} ) +ENDMACRO() +MACRO( ADD_${PROJ}_WEEKLY_TEST EXEFILE PROCS ${ARGN} ) + CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 TRUE FALSE ${ARGN} ) +ENDMACRO() MACRO( ADD_${PROJ}_TEST_THREAD_MPI EXEFILE PROCS THREADS ${ARGN} ) - ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} ) - CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_${THREADS}threads" ${ARGN} ) - MATH( EXPR TOT_PROCS "${PROCS} * ${THREADS}" ) - IF ( ${TOT_PROCS} GREATER ${TEST_MAX_PROCS} ) - MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})") - ELSEIF ( ( ${PROCS} STREQUAL "1" ) AND NOT USE_MPI_FOR_SERIAL_TESTS ) - ADD_TEST( NAME ${TESTNAME} COMMAND $ ${ARGN} ) - SET_TESTS_PROPERTIES ( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} ) - ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} ) - ELSEIF ( USE_MPI OR USE_EXT_MPI ) - ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $ ${ARGN} ) - SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none ) - SET_TESTS_PROPERTIES ( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} ) - ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} ) - ENDIF() + CALL_ADD_TEST( ${EXEFILE} ${PROCS} ${THREADS} FALSE FALSE ${ARGN} ) +ENDMACRO() +MACRO( ADD_${PROJ}_TEST_1_2_4 EXEFILE ${ARGN} ) + CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE FALSE ${ARGN} ) + CALL_ADD_TEST( ${EXEFILE} 2 1 FALSE FALSE ${ARGN} ) + CALL_ADD_TEST( ${EXEFILE} 4 1 FALSE FALSE ${ARGN} ) +ENDMACRO() +MACRO( ADD_TB_PROVISIONAL_TEST EXEFILE ${ARGN} ) + CALL_ADD_TEST( ${EXEFILE} 0 1 FALSE TRUE ${ARGN} ) +ENDMACRO() +MACRO( ADD_TB_TEST EXEFILE ${ARGN} ) + CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE TRUE ${ARGN} ) +ENDMACRO() +MACRO( ADD_TB_TEST_PARALLEL EXEFILE PROCS ${ARGN} ) + CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 FALSE TRUE ${ARGN} ) +ENDMACRO() +MACRO( ADD_TB_WEEKLY_TEST EXEFILE PROCS ${ARGN} ) + CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 TRUE TRUE ${ARGN} ) +ENDMACRO() +MACRO( ADD_TB_TEST_THREAD_MPI EXEFILE PROCS THREADS ${ARGN} ) + CALL_ADD_TEST( ${EXEFILE} ${PROCS} ${THREADS} FALSE TRUE ${ARGN} ) +ENDMACRO() +MACRO( ADD_TB_TEST_1_2_4 EXEFILE ${ARGN} ) + CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE TRUE ${ARGN} ) + CALL_ADD_TEST( ${EXEFILE} 2 1 FALSE TRUE ${ARGN} ) + CALL_ADD_TEST( ${EXEFILE} 4 1 FALSE TRUE ${ARGN} ) ENDMACRO() @@ -966,7 +1075,7 @@ FUNCTION( ADD_${PROJ}_EXAMPLE EXEFILE PROCS ${ARGN} ) ADD_TEST( NAME ${TESTNAME} COMMAND $ ${ARGN} ) ELSEIF ( USE_EXT_MPI AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) ) CREATE_TEST_NAME( "example--${EXEFILE}_${PROCS}procs" ${ARGN} ) - ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $ ${ARGN} ) + ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $ ${ARGN} ) SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none ) ENDIF() SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} ) @@ -1038,6 +1147,8 @@ FUNCTION( ADD_MATLAB_MEX SOURCE ) MATLAB_ADD_MEX( NAME ${TARGET} SRC ${SOURCE} + MODULE + R2017b ) TARGET_LINK_LIBRARIES( ${TARGET} ${MATLAB_TARGET} ) ADD_PROJ_EXE_DEP( ${TARGET} ) @@ -1074,9 +1185,11 @@ ENDMACRO() # Create a script to start matlab preloading libraries FUNCTION( CREATE_MATLAB_WRAPPER ) + SET( MATLABPATH ${MATLABPATH} "${${PROJ}_INSTALL_DIR}/mex" ) SET( tmp_libs ${MEX_LIBCXX} ${MEX_FILES} ) - STRING(REGEX REPLACE ";" ":" tmp_libs "${tmp_libs}") - STRING(REGEX REPLACE ";" ":" tmp_path "${MATLABPATH}") + LIST( REMOVE_DUPLICATES MATLABPATH ) + STRING( REGEX REPLACE ";" ":" tmp_libs "${tmp_libs}" ) + STRING( REGEX REPLACE ";" ":" tmp_path "${MATLABPATH}" ) IF ( USING_MSVC ) # Create a matlab wrapper for windows SET( MATLAB_GUI "${CMAKE_CURRENT_BINARY_DIR}/tmp/matlab-gui.bat" ) @@ -1185,7 +1298,7 @@ MACRO( ADD_DISTCLEAN ${ARGN} ) compile_commands.json ${ARGN} ) - ADD_CUSTOM_TARGET (distclean @echo cleaning for source distribution) + ADD_CUSTOM_TARGET(distclean @echo cleaning for source distribution) IF (UNIX) ADD_CUSTOM_COMMAND( DEPENDS clean @@ -1269,3 +1382,4 @@ FUNCTION( FIND_PYTHON_MODULE MODULE) ENDIF() SET( PY_${MODULE2}_FOUND ${PY_${MODULE2}_FOUND} PARENT_SCOPE ) ENDFUNCTION(FIND_PYTHON_MODULE) + diff --git a/hip/FreeLee.cu b/hip/FreeLee.cu index 09bc8689..0c3c9859 100644 --- a/hip/FreeLee.cu +++ b/hip/FreeLee.cu @@ -1,6 +1,8 @@ #include #include + #include "hip/hip_runtime.h" +#include "hip/hip_runtime_api.h" #define STOKES @@ -2034,7 +2036,7 @@ extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, doubl extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np) { - hipFuncSetCacheConfig(dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField, hipFuncCachePreferL1); + hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField, hipFuncCachePreferL1); dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField<<>>(neighborList, Map, hq, Den, Phi, ColorGrad, Vel, rhoA, rhoB, tauM, W, start, finish, Np); hipError_t err = hipGetLastError(); @@ -2046,7 +2048,7 @@ extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){ - hipFuncSetCacheConfig(dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField, hipFuncCachePreferL1); + hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField, hipFuncCachePreferL1); dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField<<>>( Map, hq, Den, Phi, ColorGrad, Vel, rhoA, rhoB, tauM, W, start, finish, Np); hipError_t err = hipGetLastError(); if (hipSuccess != err){ @@ -2057,7 +2059,7 @@ extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, dou extern "C" void ScaLBL_D3Q7_ComputePhaseField(int *Map, double *hq, double *Den, double *Phi, double rhoA, double rhoB, int start, int finish, int Np){ - hipFuncSetCacheConfig(dvc_ScaLBL_D3Q7_ComputePhaseField, hipFuncCachePreferL1); + hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q7_ComputePhaseField, hipFuncCachePreferL1); dvc_ScaLBL_D3Q7_ComputePhaseField<<>>( Map, hq, Den, Phi, rhoA, rhoB, start, finish, Np); hipError_t err = hipGetLastError(); if (hipSuccess != err){ @@ -2070,7 +2072,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, dou double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ - hipFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel, hipFuncCachePreferL1); + hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel, hipFuncCachePreferL1); dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel<<>>(neighborList, Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); hipError_t err = hipGetLastError(); @@ -2083,7 +2085,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ - hipFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel, hipFuncCachePreferL1); + hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel, hipFuncCachePreferL1); dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel<<>>(Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); hipError_t err = hipGetLastError(); @@ -2096,7 +2098,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborList, double *dist, double *Vel, double *Pressure, double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){ - hipFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK, hipFuncCachePreferL1); + hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK, hipFuncCachePreferL1); dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK<<>>(neighborList, dist, Vel, Pressure, tau, rho0, Fx, Fy, Fz, start, finish, Np); hipError_t err = hipGetLastError(); @@ -2108,7 +2110,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborLis extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, double *Vel, double *Pressure, double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){ - hipFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK, hipFuncCachePreferL1); + hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK, hipFuncCachePreferL1); dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK<<>>(dist, Vel, Pressure, tau, rho0, Fx, Fy, Fz, start, finish, Np); hipError_t err = hipGetLastError(); @@ -2119,4 +2121,4 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, d extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad,int strideY, int strideZ, int start, int finish, int Np){ -} \ No newline at end of file +} diff --git a/sample_scripts/configure_spock_TPLs b/sample_scripts/configure_spock_TPLs new file mode 100755 index 00000000..e5293e8a --- /dev/null +++ b/sample_scripts/configure_spock_TPLs @@ -0,0 +1,33 @@ +export TPL_ROOT=/ccs/home/mbt/repos +export TPL_BUILDER=/ccs/home/mbt/repos/TPL-builder +export TPL_WEBPAGE=http://bitbucket.org/AdvancedMultiPhysics/tpl-builder/downloads + +export INSTALL_DIR=/ccs/home/mbt/spock/install + +module load cmake +module load llvm-amdgpu +module load hip + + +cmake \ + -D CMAKE_BUILD_TYPE=Release \ + -D CXX_STD=14 \ + -D TPL_INSTALL_DIR:PATH="${INSTALL_DIR}" \ + -D C_COMPILER=cc \ + -D CXX_COMPILER=CC \ + -D Fortran_COMPILER=ftn \ + -D FFLAGS="-fPIC" \ + -D CFLAGS="-fPIC" \ + -D CXXFLAGS="-fPIC" \ + -D LDFLAGS="" \ + -D ENABLE_STATIC:BOOL=ON \ + -D ENABLE_SHARED:BOOL=OFF \ + -D PROCS_INSTALL=8 \ + -D TPL_LIST:STRING="TIMER;ZLIB;HDF5;SILO" \ + -D TIMER_URL="${TPL_ROOT}/TimerUtility" \ + -D ZLIB_URL="http://zlib.net/zlib-1.2.11.tar.gz" \ + -D HDF5_URL="${TPL_ROOT}/hdf5-1.8.12.tar.gz" \ + -D BUILD_TYPE=x86_64 \ + -D SILO_URL="${TPL_ROOT}/silo-4.10.2.tar.gz" \ + ${TPL_BUILDER} + diff --git a/sample_scripts/configure_spock_hip b/sample_scripts/configure_spock_hip new file mode 100755 index 00000000..19eb208f --- /dev/null +++ b/sample_scripts/configure_spock_hip @@ -0,0 +1,30 @@ +module load cmake +module load llvm-amdgpu +module load hip + + +export TPL_DIR=/gpfs/alpine/stf006/proj-shared/mbt/spock/install + + +# configure +rm -rf CMake* +cmake \ + -D CMAKE_BUILD_TYPE:STRING=Release \ + -D CMAKE_C_COMPILER:PATH=cc \ + -D CMAKE_CXX_COMPILER:PATH=CC \ + -D CMAKE_CXX_STANDARD=14 \ + -D USE_HIP=1 \ + -D LINK_LIBRARIES=${HIP_PATH}/lib/libamdhip64.so \ + -D USE_CUDA=0 \ + -D CMAKE_CUDA_FLAGS="-arch sm_70 -Xptxas=-v -Xptxas -dlcm=cg -lineinfo" \ + -D CMAKE_CUDA_HOST_COMPILER="gcc" \ + -D USE_MPI=0 \ + -D USE_HDF5=1 \ + -D HDF5_DIRECTORY="${TPL_DIR}/hdf5" \ + -D USE_SILO=0 \ + -D SILO_DIRECTORY="${TPL_DIR}/silo" \ + -D USE_DOXYGEN:BOOL=false \ + -D USE_TIMER=0 \ + ~/repos/LBPM-WIA + + From a2b22e5e47aa49acd14c907b7a373ea945120df1 Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Mon, 17 May 2021 12:00:40 -0400 Subject: [PATCH 3/4] Fixing bug in PoissonSolver --- CMakeLists.txt | 4 +++- IO/IOHelpers.h | 2 +- IO/silo.h | 7 +++++-- IO/silo.hpp | 6 ++++-- cmake/macros.cmake | 4 +++- models/PoissonSolver.cpp | 42 +++++++++++++++++++--------------------- models/PoissonSolver.h | 1 - 7 files changed, 36 insertions(+), 30 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a45f71f5..31518d86 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,8 @@ # Set some CMake properties CMAKE_MINIMUM_REQUIRED( VERSION 3.9 ) -CMAKE_POLICY( SET CMP0115 OLD ) +if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.20.0") + CMAKE_POLICY( SET CMP0115 OLD ) +endif() MESSAGE("====================") diff --git a/IO/IOHelpers.h b/IO/IOHelpers.h index 4389c619..a7ba6096 100644 --- a/IO/IOHelpers.h +++ b/IO/IOHelpers.h @@ -57,6 +57,6 @@ inline std::vector splitList( const char *line, const char token ) } -}; // namespace IO +} // namespace IO #endif diff --git a/IO/silo.h b/IO/silo.h index 5e1068fe..a3edfeb7 100644 --- a/IO/silo.h +++ b/IO/silo.h @@ -18,7 +18,8 @@ typedef int DBfile; #endif -namespace IO::silo { +namespace IO { +namespace silo { enum FileMode { READ, WRITE, CREATE }; @@ -256,7 +257,9 @@ void writeMultiVar( DBfile *fid, const std::string &varname, const std::vector &subVarNames, const std::vector &subVarTypes ); -}; // namespace IO::silo +} // namespace silo +} // namespace IO + #endif #include "IO/silo.hpp" diff --git a/IO/silo.hpp b/IO/silo.hpp index b76ebd28..72ff7041 100644 --- a/IO/silo.hpp +++ b/IO/silo.hpp @@ -13,7 +13,8 @@ #include -namespace IO::silo { +namespace IO { +namespace silo { /**************************************************** @@ -413,7 +414,8 @@ Array readTriMeshVariable( DBfile *fid, const std::string &varname ) } -}; // namespace IO::silo +} // namespace silo +} // namespace IO #endif diff --git a/cmake/macros.cmake b/cmake/macros.cmake index c8da0a45..f0ed8c67 100644 --- a/cmake/macros.cmake +++ b/cmake/macros.cmake @@ -1,6 +1,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.18.3) CMAKE_POLICY( SET CMP0057 NEW ) -CMAKE_POLICY( SET CMP0115 OLD ) +if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.20.0") + CMAKE_POLICY( SET CMP0115 OLD ) +endif() INCLUDE(CheckCCompilerFlag) INCLUDE(CheckCSourceCompiles) diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index 25a31600..739b16bf 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -218,20 +218,19 @@ void ScaLBL_Poisson::ReadInput(){ void ScaLBL_Poisson::AssignSolidBoundary(double *poisson_solid) { - size_t NLABELS=0; signed char VALUE=0; double AFFINITY=0.f; auto LabelList = electric_db->getVector( "SolidLabels" ); auto AffinityList = electric_db->getVector( "SolidValues" ); - NLABELS=LabelList.size(); + size_t NLABELS = LabelList.size(); if (NLABELS != AffinityList.size()){ ERROR("Error: LB-Poisson Solver: SolidLabels and SolidValues must be the same length! \n"); } - double label_count[NLABELS]; - double label_count_global[NLABELS]; + std::vector label_count( NLABELS, 0.0 ); + std::vector label_count_global( NLABELS, 0.0 ); // Assign the labels for (size_t idx=0; idx db0); From 3cdfb7caf6c0cd31d68e38e9d7156d5a8f5e8603 Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Mon, 17 May 2021 12:26:29 -0400 Subject: [PATCH 4/4] Minor changes to PoissonSolver --- models/PoissonSolver.cpp | 54 +++++++++++++++++++++------------------- models/PoissonSolver.h | 2 ++ 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index 739b16bf..54d3d118 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -5,19 +5,35 @@ #include "analysis/distance.h" #include "common/ReadMicroCT.h" -ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, const Utilities::MPI& COMM): -rank(RANK), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),tolerance(0),h(0), -epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Vin(0),Vout(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0), -chargeDen_dummy(0),WriteLog(0),nprocx(0),nprocy(0),nprocz(0), -BoundaryConditionInlet(0),BoundaryConditionOutlet(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0), -Vin0(0),freqIn(0),t0_In(0),Vin_Type(0),Vout0(0),freqOut(0),t0_Out(0),Vout_Type(0), -TestPeriodic(0),TestPeriodicTime(0),TestPeriodicTimeConv(0),TestPeriodicSaveInterval(0), -comm(COMM) + +static inline bool fileExists( const std::string &filename ) { - + std::ifstream ifile( filename.c_str() ); + return ifile.good(); } -ScaLBL_Poisson::~ScaLBL_Poisson(){ + +ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, const Utilities::MPI& COMM): + rank(RANK), TIMELOG(nullptr), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),tolerance(0),h(0), + epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Vin(0),Vout(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0), + chargeDen_dummy(0),WriteLog(0),nprocx(0),nprocy(0),nprocz(0), + BoundaryConditionInlet(0),BoundaryConditionOutlet(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0), + Vin0(0),freqIn(0),t0_In(0),Vin_Type(0),Vout0(0),freqOut(0),t0_Out(0),Vout_Type(0), + TestPeriodic(0),TestPeriodicTime(0),TestPeriodicTimeConv(0),TestPeriodicSaveInterval(0), + comm(COMM) +{ + if ( rank == 0 ) { + bool WriteHeader = !fileExists( "PoissonSolver_Convergence.csv" ); + + TIMELOG = fopen("PoissonSolver_Convergence.csv","a+"); + if (WriteHeader) + fprintf(TIMELOG,"Timestep Error\n"); + } +} +ScaLBL_Poisson::~ScaLBL_Poisson() +{ + if ( TIMELOG ) + fclose( TIMELOG ); } void ScaLBL_Poisson::ReadParams(string filename){ @@ -596,24 +612,10 @@ void ScaLBL_Poisson::Run(double *ChargeDensity, int timestep_from_Study){ } -static inline bool fileExists( const std::string &filename ) -{ - std::ifstream ifile( filename.c_str() ); - return ifile.good(); -} - - void ScaLBL_Poisson::getConvergenceLog(int timestep,double error){ if ( rank == 0 ) { - bool WriteHeader = !fileExists( "PoissonSolver_Convergence.csv" ); - - auto fid = fopen("PoissonSolver_Convergence.csv","a+"); - if (WriteHeader) - fprintf(fid,"Timestep Error\n"); - - fprintf(fid,"%i %.5g\n",timestep,error); - fflush(fid); - fclose( fid ); + fprintf(TIMELOG,"%i %.5g\n",timestep,error); + fflush(TIMELOG); } } diff --git a/models/PoissonSolver.h b/models/PoissonSolver.h index f4941159..2ffbd1a7 100644 --- a/models/PoissonSolver.h +++ b/models/PoissonSolver.h @@ -89,6 +89,8 @@ public: private: Utilities::MPI comm; + FILE *TIMELOG; + // filenames char LocalRankString[8]; char LocalRankFilename[40];