OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'FOM'

Browse Source
This commit is contained in:
James McClure 2021-05-17 14:10:56 -04:00
commit 07af06a7cd
11 changed files with 360 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@ -1,5 +1,8 @@
# Set some CMake properties
CMAKE_MINIMUM_REQUIRED( VERSION 3.9 )
if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.20.0")
CMAKE_POLICY( SET CMP0115 OLD )
endif()
MESSAGE("====================")

2
IO/IOHelpers.h
View File

@ -57,6 +57,6 @@ inline std::vector<std::string> splitList( const char *line, const char token )
}
}; // namespace IO
} // namespace IO
#endif

7
IO/silo.h
View File

@ -18,7 +18,8 @@ typedef int DBfile;
#endif
namespace IO::silo {
namespace IO {
namespace silo {
enum FileMode { READ, WRITE, CREATE };
@ -256,7 +257,9 @@ void writeMultiVar( DBfile *fid, const std::string &varname,
const std::vector<std::string> &subVarNames, const std::vector<int> &subVarTypes );
}; // namespace IO::silo
} // namespace silo
} // namespace IO
#endif
#include "IO/silo.hpp"

6
IO/silo.hpp
View File

@ -13,7 +13,8 @@
#include <silo.h>
namespace IO::silo {
namespace IO {
namespace silo {
/****************************************************
@ -413,7 +414,8 @@ Array<TYPE> readTriMeshVariable( DBfile *fid, const std::string &varname )
}
}; // namespace IO::silo
} // namespace silo
} // namespace IO
#endif

338
cmake/macros.cmake
View File

@ -1,3 +1,9 @@
CMAKE_MINIMUM_REQUIRED(VERSION 3.18.3)
CMAKE_POLICY( SET CMP0057 NEW )
if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.20.0")
CMAKE_POLICY( SET CMP0115 OLD )
endif()
INCLUDE(CheckCCompilerFlag)
INCLUDE(CheckCSourceCompiles)
INCLUDE(CheckCXXCompilerFlag)
@ -47,12 +53,24 @@ ELSEIF( NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION )
ENDIF()
# Check for gold linker
IF ( UNIX AND NOT APPLE AND NOT DISABLE_GOLD )
EXECUTE_PROCESS( COMMAND ${CMAKE_CXX_COMPILER} -fuse-ld=gold -Wl,--version ERROR_QUIET OUTPUT_VARIABLE LD_VERSION )
IF ( "${LD_VERSION}" MATCHES "GNU gold" )
MESSAGE( "Using gold linker" )
SET( GOLD_LINK_FLAGS "-fuse-ld=gold -Wl,--disable-new-dtags -Wl,--no-map-whole-files" )
SET( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GOLD_LINK_FLAGS}" )
SET( CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${GOLD_LINK_FLAGS}" )
ENDIF()
ENDIF()
# Add some default targets if they do not exist
IF ( NOT TARGET copy-${PROJ}-Data )
ADD_CUSTOM_TARGET( copy-${PROJ}-Data ALL )
ENDIF()
IF ( NOT TARGET copy-${PROJ}-include )
ADD_CUSTOM_TARGET ( copy-${PROJ}-include ALL )
ADD_CUSTOM_TARGET( copy-${PROJ}-include ALL )
ENDIF()
@ -75,7 +93,7 @@ ENDMACRO()
MACRO( PRINT_ALL_VARIABLES )
GET_CMAKE_PROPERTY(_variableNames VARIABLES)
FOREACH(_variableName ${_variableNames})
message(STATUS "${_variableName}=${${_variableName}}")
MESSAGE(STATUS "${_variableName}=${${_variableName}}")
ENDFOREACH()
ENDMACRO()
@ -108,28 +126,28 @@ MACRO( CONVERT_M4_FORTRAN IN LOCAL_PATH OUT_PATH )
IF ( NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/${OUT_PATH}" )
FILE(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${OUT_PATH}" )
ENDIF()
CONFIGURE_FILE ( ${IN} ${IN} COPYONLY )
CONFIGURE_FILE( ${IN} ${IN} COPYONLY )
IF ("${CMAKE_GENERATOR}" STREQUAL "Xcode")
STRING(REGEX REPLACE ".F" ".o" OUT2 "${OUT}" )
STRING(REGEX REPLACE ";" " " COMPILE_CMD "${CMAKE_Fortran_COMPILER} -c ${OUT} ${CMAKE_Fortran_FLAGS} -o ${OUT2}")
STRING(REGEX REPLACE "\\\\" "" COMPILE_CMD "${COMPILE_CMD}")
MESSAGE("COMPILE_CMD = ${COMPILE_CMD}")
SET( COMPILE_CMD ${COMPILE_CMD} )
add_custom_command(
ADD_CUSTOM_COMMAND(
OUTPUT ${OUT2}
COMMAND m4 -I${LOCAL_PATH} -I${SAMRAI_FORTDIR} ${M4DIRS} ${IN} > ${OUT}
COMMAND ${COMPILE_CMD}
DEPENDS ${IN}
)
set_source_files_properties(${OUT2} PROPERTIES GENERATED true)
SET_SOURCE_FILES_PROPERTIES(${OUT2} PROPERTIES GENERATED true)
SET( SOURCES ${SOURCES} "${OUT2}" )
ELSE()
add_custom_command(
ADD_CUSTOM_COMMAND(
OUTPUT ${OUT}
COMMAND m4 -I${LOCAL_PATH} -I${SAMRAI_FORTDIR} ${M4DIRS} ${M4_OPTIONS} ${IN} > ${OUT}
DEPENDS ${IN}
)
set_source_files_properties(${OUT} PROPERTIES GENERATED true)
SET_SOURCE_FILES_PROPERTIES(${OUT} PROPERTIES GENERATED true)
SET( SOURCES ${SOURCES} "${OUT}" )
ENDIF()
ENDMACRO()
@ -145,6 +163,10 @@ ENDMACRO()
# Add a project executable
MACRO( ADD_${PROJ}_EXECUTABLE EXEFILE )
IF ( TARGET ${EXEFILE} AND NOT INCLUDE_${EXEFILE} )
MESSAGE( FATAL_ERROR "ADD_${PROJ}_EXECUTABLE should be called before adding tests" )
ENDIF()
SET( INCLUDE_${EXEFILE} TRUE )
ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
INSTALL( TARGETS ${EXEFILE} DESTINATION ${${PROJ}_INSTALL_DIR}/bin )
ENDMACRO()
@ -164,7 +186,7 @@ ENDMACRO ()
# Find the source files
MACRO (FIND_FILES)
MACRO( FIND_FILES )
# Find the C/C++ headers
SET( T_HEADERS "" )
FILE( GLOB T_HEADERS "*.h" "*.H" "*.hh" "*.hpp" "*.I" )
@ -193,7 +215,7 @@ MACRO (FIND_FILES)
SET( CSOURCES ${CSOURCES} ${T_CSOURCES} )
SET( FSOURCES ${FSOURCES} ${T_FSOURCES} )
SET( M4FSOURCES ${M4FSOURCES} ${T_M4FSOURCES} )
SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${T_M4FSOURCES} )
SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${T_M4FSOURCES} ${CUDASOURCES} )
ENDMACRO()
@ -303,7 +325,7 @@ MACRO( INSTALL_${PROJ}_TARGET PACKAGE )
IF ( TARGET write_repo_version )
ADD_DEPENDENCIES( ${PACKAGE} write_repo_version )
ENDIF()
ADD_DEPENDENCIES ( ${PACKAGE} copy-${PROJ}-include )
ADD_DEPENDENCIES( ${PACKAGE} copy-${PROJ}-include )
IF ( NOT ${PROJ}_LIB )
INSTALL( TARGETS ${PACKAGE} DESTINATION ${${PROJ}_INSTALL_DIR}/lib )
ENDIF()
@ -340,7 +362,7 @@ ENDMACRO()
# Macro to verify that a path has been set
MACRO( VERIFY_PATH PATH_NAME )
IF ("${PATH_NAME}" STREQUAL "")
MESSAGE ( FATAL_ERROR "Path is not set: ${PATH_NAME}" )
MESSAGE( FATAL_ERROR "Path is not set: ${PATH_NAME}" )
ENDIF()
IF ( NOT EXISTS "${PATH_NAME}" )
MESSAGE( FATAL_ERROR "Path does not exist: ${PATH_NAME}" )
@ -420,6 +442,9 @@ MACRO( IDENTIFY_COMPILER )
ELSEIF ( ${CMAKE_Fortran_COMPILER_ID} MATCHES "PGI")
SET(USING_PGF90 TRUE)
MESSAGE("Using pgf90")
ELSEIF( (${CMAKE_Fortran_COMPILER_ID} MATCHES "CRAY") OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "Cray") )
SET(USING_CRAY TRUE)
MESSAGE("Using Cray")
ELSEIF ( (${CMAKE_Fortran_COMPILER_ID} MATCHES "CLANG") OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "Clang") OR
(${CMAKE_Fortran_COMPILER_ID} MATCHES "FLANG") OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "Flang") )
SET(USING_FLANG TRUE)
@ -440,7 +465,7 @@ MACRO( SET_WARNINGS )
# Note: adding -Wlogical-op causes a wierd linking error on Titan using the nvcc wrapper:
# /usr/bin/ld: cannot find gical-op: No such file or directory
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Woverloaded-virtual -Wsign-compare")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -Woverloaded-virtual -Wsign-compare -Wformat-security")
ELSEIF ( USING_MSVC )
# Add Microsoft specifc compiler options
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /D _SCL_SECURE_NO_WARNINGS /D _CRT_SECURE_NO_WARNINGS /D _ITERATOR_DEBUG_LEVEL=0 /wd4267" )
@ -448,7 +473,7 @@ MACRO( SET_WARNINGS )
ELSEIF ( USING_ICC )
# Add Intel specifc compiler options
SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -Wall" )
SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall" )
SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall -wd1011" )
ELSEIF ( USING_CRAY )
# Add default compiler options
SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS}")
@ -461,8 +486,8 @@ MACRO( SET_WARNINGS )
SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} --diag_suppress 111,128,185")
ELSEIF ( USING_CLANG )
# Add default compiler options
SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -Wall")
SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall -Wno-missing-braces -Wmissing-field-initializers -ftemplate-depth=1024")
SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -Wall -Wextra")
SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic -Wno-missing-braces -Wmissing-field-initializers -ftemplate-depth=1024")
ELSEIF ( USING_XL )
# Add default compiler options
SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -Wall")
@ -651,7 +676,7 @@ MACRO( COPY_MESH_FILE MESHNAME )
ENDFOREACH()
# We have either found the mesh or failed
IF ( NOT MESHPATH )
MESSAGE ( WARNING "Cannot find mesh: " ${MESHNAME} )
MESSAGE( WARNING "Cannot find mesh: " ${MESHNAME} )
ELSE ()
SET( MESHPATH2 )
FOREACH( tmp ${MESHPATH} )
@ -666,25 +691,16 @@ ENDMACRO()
# Link the libraries to the given target
MACRO( TARGET_LINK_EXTERNAL_LIBRARIES TARGET_NAME )
FOREACH ( tmp ${TPL_LIBS} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
# Include external libraries
FOREACH ( tmp ${EXTERNAL_LIBS} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
FOREACH ( tmp ${LAPACK_LIBS} )
# Include libraries found through the TPL builder
FOREACH ( tmp ${TPL_LIBS} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
FOREACH ( tmp ${BLAS_LIBS} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
FOREACH ( tmp ${BLAS_LAPACK_LIBS} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
FOREACH ( tmp ${MPI_LIBRARIES} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
FOREACH ( tmp ${CMAKE_C_IMPLICIT_LINK_LIBRARIES}
# Include CMake implicit libraries
FOREACH ( tmp ${CMAKE_CUDA_IMPLICIT_LINK_LIBRARIES} ${CMAKE_C_IMPLICIT_LINK_LIBRARIES}
${CMAKE_CXX_IMPLICIT_LINK_LIBRARIES} ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
@ -715,8 +731,8 @@ ENDFUNCTION()
MACRO( ADD_PROJ_EXE_DEP EXE )
# Add the package dependencies
IF( ${PROJ}_TEST_LIB_EXISTS )
ADD_DEPENDENCIES ( ${EXE} ${PACKAGE_TEST_LIB} )
TARGET_LINK_LIBRARIES ( ${EXE} ${PACKAGE_TEST_LIB} )
ADD_DEPENDENCIES( ${EXE} ${PACKAGE_TEST_LIB} )
TARGET_LINK_LIBRARIES( ${EXE} ${PACKAGE_TEST_LIB} )
ENDIF()
# Add the executable to the dependencies of check and build-test
ADD_DEPENDENCIES( check ${EXE} )
@ -783,7 +799,7 @@ FUNCTION( ADD_PROJ_PROVISIONAL_TEST EXEFILE )
SET( CXXFILE ${EXEFILE} )
SET( TESTS_SO_FAR ${TESTS_SO_FAR} ${EXEFILE} )
# Check if we want to add the test to all
IF ( NOT EXCLUDE_TESTS_FROM_ALL )
IF ( NOT EXCLUDE_TESTS_FROM_ALL OR INCLUDE_${EXEFILE} )
ADD_EXECUTABLE( ${EXEFILE} ${CXXFILE} )
ELSE()
ADD_EXECUTABLE( ${EXEFILE} EXCLUDE_FROM_ALL ${CXXFILE} )
@ -818,123 +834,218 @@ MACRO( CREATE_TEST_NAME TEST ${ARGN} )
FOREACH( tmp ${ARGN})
SET( TESTNAME "${TESTNAME}--${tmp}")
endforeach()
# STRING(REGEX REPLACE "--" "-" TESTNAME ${TESTNAME} )
SET( LAST_TESTNAME ${TESTNAME} PARENT_SCOPE )
ENDMACRO()
# Function to add the resource locks to an executable
FUNCTION( ADD_RESOURCE_LOCK TESTNAME EXEFILE ${ARGN} )
FUNCTION( ADD_RESOURCE TESTNAME RESOURCE )
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES RESOURCE_LOCK ${RESOURCE} )
ENDFUNCTION()
FUNCTION( ADD_RESOURCE_LOCKS TESTNAME EXEFILE ${ARGN} )
IF ( NOT DISABLE_RESOURCE_LOCK )
IF ( NOT ARGN )
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES RESOURCE_LOCK ${EXEFILE} )
ADD_RESOURCE( ${TESTNAME} ${EXEFILE} )
ELSE()
FOREACH( tmp ${ARGN} )
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES RESOURCE_LOCK ${tmp} )
ADD_RESOURCE( ${TESTNAME} ${tmp} )
ENDFOREACH()
ENDIF()
ENDIF()
ENDFUNCTION()
# Add a executable as a test
FUNCTION( ADD_${PROJ}_TEST EXEFILE ${ARGN} )
# Function to add a test
FUNCTION( CALL_ADD_TEST EXEFILE PROCS THREADS WEEKLY TESTBUILDER ${ARGN} )
# Check if we actually want to add the test
KEEP_TEST( RESULT )
IF ( NOT RESULT )
MESSAGE( "Discarding test: ${EXEFILE}" )
RETURN()
ENDIF()
# Add the provisional test
ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
CREATE_TEST_NAME( ${EXEFILE} ${ARGN} )
IF ( USE_MPI_FOR_SERIAL_TESTS )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
# Check if we are dealing with a TestBuilder test
IF ( ${TESTBUILDER} )
# Add the test to the TestBuilder sources
SET( TESTBUILDER_SOURCES ${TESTBUILDER_SOURCES} ${EXEFILE} PARENT_SCOPE )
ELSE()
ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
# Add the provisional test
ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
ENDIF()
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS 1 )
ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
ENDFUNCTION()
# Add a executable as a weekly test
FUNCTION( ADD_${PROJ}_WEEKLY_TEST EXEFILE PROCS ${ARGN} )
# Check if we actually want to add the test
KEEP_TEST( RESULT )
IF ( NOT RESULT )
RETURN()
# Create the test name
SET( TESTNAME ${EXEFILE} )
#IF ( ${TESTBUILDER} )
# SET( TESTNAME TB_${TESTNAME} )
#ENDIF()
IF ( "${PROCS}" GREATER "1")
SET( TESTNAME ${TESTNAME}_${PROCS}procs )
ENDIF()
# Add the provisional test
ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
IF( ${PROCS} STREQUAL "1" )
CREATE_TEST_NAME( "${EXEFILE}_WEEKLY" ${ARGN} )
ELSEIF( (USE_MPI OR USE_EXT_MPI) AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) )
CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_WEEKLY" ${ARGN} )
IF ( "${THREADS}" GREATER "1")
SET( TESTNAME ${TESTNAME}_${THREADS}threads )
ENDIF()
IF ( ${PROCS} GREATER ${TEST_MAX_PROCS} )
IF ( ${WEEKLY} )
SET( TESTNAME ${TESTNAME}_WEEKLY )
ENDIF()
CREATE_TEST_NAME( ${TESTNAME} ${ARGN} )
# Create the tests for ctest
MATH( EXPR TOT_PROCS "${PROCS} * ${THREADS}" )
IF ( ${TOT_PROCS} EQUAL 0 )
# Skip test (provisional)
ELSEIF ( ${TOT_PROCS} GREATER ${TEST_MAX_PROCS} )
MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})")
ELSEIF( ${PROCS} STREQUAL "1" )
CREATE_TEST_NAME( "${EXEFILE}_WEEKLY" ${ARGN} )
IF ( USE_MPI_FOR_SERIAL_TESTS )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
ELSEIF ( ( ${PROCS} STREQUAL "1" ) AND NOT USE_MPI_FOR_SERIAL_TESTS )
IF ( ${TESTBUILDER} )
ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${TB_TARGET}> ${EXEFILE} ${ARGN} )
ELSE()
ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
ENDIF()
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS 1 )
ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
ELSEIF( (USE_MPI OR USE_EXT_MPI) AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) )
CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_WEEKLY" ${ARGN} )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} )
ADD_RESOURCE_LOCKS( ${TESTNAME} ${EXEFILE} ${ARGN} )
ELSEIF ( USE_MPI OR USE_EXT_MPI )
IF ( ${TESTBUILDER} )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${TB_TARGET}> ${EXEFILE} ${ARGN} )
ELSE()
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
ENDIF()
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} )
ADD_RESOURCE_LOCKS( ${TESTNAME} ${EXEFILE} ${ARGN} )
ENDIF()
ENDFUNCTION()
# Add a executable as a parallel test
FUNCTION( ADD_${PROJ}_TEST_PARALLEL EXEFILE PROCS ${ARGN} )
# Functions to create a test builder
FUNCTION( INITIALIZE_TESTBUILDER )
SET( TESTBUILDER_SOURCES PARENT_SCOPE )
# Check if we actually want to add the test
KEEP_TEST( RESULT )
IF ( NOT RESULT )
RETURN()
ENDIF()
# Add the provisional test
ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs" ${ARGN} )
IF ( NOT ( USE_MPI OR USE_EXT_MPI ) )
MESSAGE("Disabling test ${TESTNAME} (configured without MPI)")
ELSEIF ( ${PROCS} GREATER ${TEST_MAX_PROCS} )
MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})")
ELSE()
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )
ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
# Check if the target does not exist (may not be added yet or we are re-configuring)
STRING(REGEX REPLACE "${${PROJ}_SOURCE_DIR}/" "" TB_TARGET "TB-${PROJ}-${CMAKE_CURRENT_SOURCE_DIR}" )
STRING(REGEX REPLACE "/" "-" TB_TARGET ${TB_TARGET} )
IF ( NOT TARGET ${TB_TARGET} )
GLOBAL_SET( ${TB_TARGET}-BINDIR )
ENDIF()
# Check if test has already been added
IF ( NOT ${TB_TARGET}-BINDIR )
GLOBAL_SET( ${TB_TARGET}-BINDIR "${CMAKE_CURRENT_BINARY_DIR}" )
# Create the initial file
IF ( NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" )
FILE( WRITE "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" "// Auto generated file\n\n" )
ENDIF()
# The target has not been added
SET( CXXFILE "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" )
SET( TESTS_SO_FAR ${TESTS_SO_FAR} ${TB_TARGET} )
# Check if we want to add the test to all
IF ( NOT EXCLUDE_TESTS_FROM_ALL )
ADD_EXECUTABLE( ${TB_TARGET} ${CXXFILE} )
ELSE()
ADD_EXECUTABLE( ${TB_TARGET} EXCLUDE_FROM_ALL ${CXXFILE} )
ENDIF()
SET_TARGET_PROPERTIES( ${TB_TARGET} PROPERTIES OUTPUT_NAME "TB" )
ELSEIF ( NOT ${TB-BINDIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR} )
# We are trying to add 2 different tests with the same name
MESSAGE( "Existing test: ${TB-BINDIR}/${TB_TARGET}" )
MESSAGE( "New test: ${CMAKE_CURRENT_BINARY_DIR}/${TB_TARGET}}" )
MESSAGE( FATAL_ERROR "Trying to add 2 different tests with the same name" )
ENDIF()
SET( TB_TARGET ${TB_TARGET} PARENT_SCOPE )
ENDFUNCTION()
FUNCTION( FINALIZE_TESTBUILDER )
# Check if we actually want to add the test
KEEP_TEST( RESULT )
IF ( NOT RESULT )
RETURN()
ENDIF()
# Create the library for the test builder
IF ( TESTBUILDER_SOURCES )
LIST( REMOVE_DUPLICATES TESTBUILDER_SOURCES )
SET( TB_TARGET_SOURCES "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" )
FOREACH ( tmp ${TESTBUILDER_SOURCES} )
GET_FILENAME_COMPONENT( tmp ${tmp} ABSOLUTE )
SET( TB_TARGET_SOURCES ${TB_TARGET_SOURCES} ${tmp} )
ENDFOREACH()
TARGET_SOURCES( ${TB_TARGET} PUBLIC ${TB_TARGET_SOURCES} )
ADD_DEPENDENCIES( ${TB_TARGET} copy-${PROJ}-include )
ENDIF()
# Add the dependencies to the test builder
ADD_PROJ_EXE_DEP( ${TB_TARGET} )
IF ( CURRENT_LIBRARY )
IF ( NOT TARGET ${CURRENT_LIBRARY}-test )
ADD_CUSTOM_TARGET( ${CURRENT_LIBRARY}-test )
ENDIF()
ADD_DEPENDENCIES( ${CURRENT_LIBRARY}-test ${TB_TARGET} )
ENDIF()
SET( TB_TARGET ${TB_TARGET} PARENT_SCOPE )
# Finish generating TestBuilder.cpp
SET( TB_FILE "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder-tmp.cpp" )
FILE( WRITE "${TB_FILE}" "// Auto generated file\n\n" )
FILE( APPEND "${TB_FILE}" "#include <cstring>\n" )
FILE( APPEND "${TB_FILE}" "#include <iostream>\n\n" )
FOREACH( tmp ${TESTBUILDER_SOURCES} )
FILE( APPEND "${TB_FILE}" "extern int ${tmp}( int argc, char *argv[] );\n" )
ENDFOREACH()
FILE( APPEND "${TB_FILE}" "\n\n// Main Program\n" )
FILE( APPEND "${TB_FILE}" "int main( int argc, char *argv[] )\n" )
FILE( APPEND "${TB_FILE}" "{\n" )
FILE( APPEND "${TB_FILE}" " if ( argc < 2 ) {\n" )
FILE( APPEND "${TB_FILE}" " std::cerr << \"Invaild number of arguments for TestBuilder\";\n " )
FILE( APPEND "${TB_FILE}" " return 1;\n" )
FILE( APPEND "${TB_FILE}" " }\n\n" )
FILE( APPEND "${TB_FILE}" " int rtn = 0;\n" )
FILE( APPEND "${TB_FILE}" " if ( strcmp( argv[1], \"NULL\" ) == 0 ) {\n" )
FOREACH( tmp ${TESTBUILDER_SOURCES} )
FILE( APPEND "${TB_FILE}" " } else if ( strcmp( argv[1], \"${tmp}\" ) == 0 ) {\n" )
FILE( APPEND "${TB_FILE}" " rtn = ${tmp}( argc-1, &argv[1] );\n" )
ENDFOREACH()
FILE( APPEND "${TB_FILE}" " } else {\n" )
FILE( APPEND "${TB_FILE}" " std::cerr << \"Unknown test\";\n" )
FILE( APPEND "${TB_FILE}" " return 1;\n" )
FILE( APPEND "${TB_FILE}" " }\n\n" )
FILE( APPEND "${TB_FILE}" " return rtn;\n" )
FILE( APPEND "${TB_FILE}" "}\n" )
EXECUTE_PROCESS( COMMAND ${CMAKE_COMMAND} -E copy_if_different "${TB_FILE}" "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" )
ENDFUNCTION()
# Add a parallel test that may use both MPI and threads
# This allows us to correctly compute the number of processors used by the test
# Convience functions to add a test
MACRO( ADD_${PROJ}_TEST EXEFILE ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE FALSE ${ARGN} )
ENDMACRO()
MACRO( ADD_${PROJ}_TEST_PARALLEL EXEFILE PROCS ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 FALSE FALSE ${ARGN} )
ENDMACRO()
MACRO( ADD_${PROJ}_WEEKLY_TEST EXEFILE PROCS ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 TRUE FALSE ${ARGN} )
ENDMACRO()
MACRO( ADD_${PROJ}_TEST_THREAD_MPI EXEFILE PROCS THREADS ${ARGN} )
ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_${THREADS}threads" ${ARGN} )
MATH( EXPR TOT_PROCS "${PROCS} * ${THREADS}" )
IF ( ${TOT_PROCS} GREATER ${TEST_MAX_PROCS} )
MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})")
ELSEIF ( ( ${PROCS} STREQUAL "1" ) AND NOT USE_MPI_FOR_SERIAL_TESTS )
ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_TESTS_PROPERTIES ( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} )
ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
ELSEIF ( USE_MPI OR USE_EXT_MPI )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
SET_TESTS_PROPERTIES ( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} )
ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
ENDIF()
CALL_ADD_TEST( ${EXEFILE} ${PROCS} ${THREADS} FALSE FALSE ${ARGN} )
ENDMACRO()
MACRO( ADD_${PROJ}_TEST_1_2_4 EXEFILE ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE FALSE ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} 2 1 FALSE FALSE ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} 4 1 FALSE FALSE ${ARGN} )
ENDMACRO()
MACRO( ADD_TB_PROVISIONAL_TEST EXEFILE ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} 0 1 FALSE TRUE ${ARGN} )
ENDMACRO()
MACRO( ADD_TB_TEST EXEFILE ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE TRUE ${ARGN} )
ENDMACRO()
MACRO( ADD_TB_TEST_PARALLEL EXEFILE PROCS ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 FALSE TRUE ${ARGN} )
ENDMACRO()
MACRO( ADD_TB_WEEKLY_TEST EXEFILE PROCS ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 TRUE TRUE ${ARGN} )
ENDMACRO()
MACRO( ADD_TB_TEST_THREAD_MPI EXEFILE PROCS THREADS ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} ${PROCS} ${THREADS} FALSE TRUE ${ARGN} )
ENDMACRO()
MACRO( ADD_TB_TEST_1_2_4 EXEFILE ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE TRUE ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} 2 1 FALSE TRUE ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} 4 1 FALSE TRUE ${ARGN} )
ENDMACRO()
@ -966,7 +1077,7 @@ FUNCTION( ADD_${PROJ}_EXAMPLE EXEFILE PROCS ${ARGN} )
ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
ELSEIF ( USE_EXT_MPI AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) )
CREATE_TEST_NAME( "example--${EXEFILE}_${PROCS}procs" ${ARGN} )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
ENDIF()
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )
@ -1038,6 +1149,8 @@ FUNCTION( ADD_MATLAB_MEX SOURCE )
MATLAB_ADD_MEX(
NAME ${TARGET}
SRC ${SOURCE}
MODULE
R2017b
)
TARGET_LINK_LIBRARIES( ${TARGET} ${MATLAB_TARGET} )
ADD_PROJ_EXE_DEP( ${TARGET} )
@ -1074,9 +1187,11 @@ ENDMACRO()
# Create a script to start matlab preloading libraries
FUNCTION( CREATE_MATLAB_WRAPPER )
SET( MATLABPATH ${MATLABPATH} "${${PROJ}_INSTALL_DIR}/mex" )
SET( tmp_libs ${MEX_LIBCXX} ${MEX_FILES} )
STRING(REGEX REPLACE ";" ":" tmp_libs "${tmp_libs}")
STRING(REGEX REPLACE ";" ":" tmp_path "${MATLABPATH}")
LIST( REMOVE_DUPLICATES MATLABPATH )
STRING( REGEX REPLACE ";" ":" tmp_libs "${tmp_libs}" )
STRING( REGEX REPLACE ";" ":" tmp_path "${MATLABPATH}" )
IF ( USING_MSVC )
# Create a matlab wrapper for windows
SET( MATLAB_GUI "${CMAKE_CURRENT_BINARY_DIR}/tmp/matlab-gui.bat" )
@ -1185,7 +1300,7 @@ MACRO( ADD_DISTCLEAN ${ARGN} )
compile_commands.json
${ARGN}
)
ADD_CUSTOM_TARGET (distclean @echo cleaning for source distribution)
ADD_CUSTOM_TARGET(distclean @echo cleaning for source distribution)
IF (UNIX)
ADD_CUSTOM_COMMAND(
DEPENDS clean
@ -1269,3 +1384,4 @@ FUNCTION( FIND_PYTHON_MODULE MODULE)
ENDIF()
SET( PY_${MODULE2}_FOUND ${PY_${MODULE2}_FOUND} PARENT_SCOPE )
ENDFUNCTION(FIND_PYTHON_MODULE)

10
common/MPI.I
View File

@ -595,7 +595,15 @@ template<class TYPE>
void MPI_CLASS::sendrecv( const TYPE *sendbuf, int sendcount, int dest, int sendtag,
TYPE *recvbuf, int recvcount, int source, int recvtag ) const
{
ERROR( "Not implimented" );
if ( getSize() == 1 ) {
ASSERT( dest == 0 );
ASSERT( source == 0 );
ASSERT( sendcount == recvcount );
ASSERT( sendtag == recvtag );
memcpy( recvbuf, sendbuf, sendcount * sizeof( TYPE ) );
} else {
ERROR( "Not implimented for " + std::string( typeid( TYPE ).name() ) );
}
}
#else
template<class TYPE>

22
hip/FreeLee.cu
View File

@ -1,6 +1,7 @@
#include <math.h>
#include <stdio.h>
#include <cuda_profiler_api.h>
#include "hip/hip_runtime.h"
#include "hip/hip_runtime_api.h"
#define STOKES
@ -3395,7 +3396,8 @@ extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, doubl
extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np)
{
cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField, cudaFuncCachePreferL1);
hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField, hipFuncCachePreferL1);
dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField<<<NBLOCKS,NTHREADS >>>(neighborList, Map, hq, Den, Phi, ColorGrad, Vel,
rhoA, rhoB, tauM, W, start, finish, Np);
hipError_t err = hipGetLastError();
@ -3407,7 +3409,8 @@ extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map
extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){
cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField, cudaFuncCachePreferL1);
hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField, hipFuncCachePreferL1);

dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField<<<NBLOCKS,NTHREADS >>>( Map, hq, Den, Phi, ColorGrad, Vel, rhoA, rhoB, tauM, W, start, finish, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
@ -3418,7 +3421,7 @@ extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, dou
extern "C" void ScaLBL_D3Q7_ComputePhaseField(int *Map, double *hq, double *Den, double *Phi, double rhoA, double rhoB, int start, int finish, int Np){
cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q7_ComputePhaseField, cudaFuncCachePreferL1);
hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q7_ComputePhaseField, hipFuncCachePreferL1);
dvc_ScaLBL_D3Q7_ComputePhaseField<<<NBLOCKS,NTHREADS >>>( Map, hq, Den, Phi, rhoA, rhoB, start, finish, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
@ -3431,7 +3434,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, dou
double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz,
int strideY, int strideZ, int start, int finish, int Np){
cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel, cudaFuncCachePreferL1);
hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel, hipFuncCachePreferL1);
dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel<<<NBLOCKS,NTHREADS >>>(neighborList, Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad,
rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np);
hipError_t err = hipGetLastError();
@ -3444,7 +3447,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double
double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz,
int strideY, int strideZ, int start, int finish, int Np){
cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel, cudaFuncCachePreferL1);
hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel, hipFuncCachePreferL1);
dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel<<<NBLOCKS,NTHREADS >>>(Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad,
rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np);
hipError_t err = hipGetLastError();
@ -3502,7 +3505,7 @@ extern "C" void ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(int *Map, double *hq,
extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborList, double *dist, double *Vel, double *Pressure,
double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){
cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK, cudaFuncCachePreferL1);
hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK, hipFuncCachePreferL1);
dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK<<<NBLOCKS,NTHREADS >>>(neighborList, dist, Vel, Pressure,
tau, rho0, Fx, Fy, Fz, start, finish, Np);
hipError_t err = hipGetLastError();
@ -3514,7 +3517,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborLis
extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, double *Vel, double *Pressure,
double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){
cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK, cudaFuncCachePreferL1);
hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK, hipFuncCachePreferL1);
dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK<<<NBLOCKS,NTHREADS >>>(dist, Vel, Pressure,
tau, rho0, Fx, Fy, Fz, start, finish, Np);
hipError_t err = hipGetLastError();
@ -3525,4 +3529,4 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, d
extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad,int strideY, int strideZ, int start, int finish, int Np){
}
}

66
models/PoissonSolver.cpp
View File

@ -5,19 +5,35 @@
#include "analysis/distance.h"
#include "common/ReadMicroCT.h"
ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, const Utilities::MPI& COMM):
rank(RANK), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),tolerance(0),h(0),
epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Vin(0),Vout(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0),
chargeDen_dummy(0),WriteLog(0),nprocx(0),nprocy(0),nprocz(0),
BoundaryConditionInlet(0),BoundaryConditionOutlet(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),
Vin0(0),freqIn(0),t0_In(0),Vin_Type(0),Vout0(0),freqOut(0),t0_Out(0),Vout_Type(0),
TestPeriodic(0),TestPeriodicTime(0),TestPeriodicTimeConv(0),TestPeriodicSaveInterval(0),
comm(COMM)
static inline bool fileExists( const std::string &filename )
{
std::ifstream ifile( filename.c_str() );
return ifile.good();
}
ScaLBL_Poisson::~ScaLBL_Poisson(){
ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, const Utilities::MPI& COMM):
rank(RANK), TIMELOG(nullptr), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),tolerance(0),h(0),
epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Vin(0),Vout(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0),
chargeDen_dummy(0),WriteLog(0),nprocx(0),nprocy(0),nprocz(0),
BoundaryConditionInlet(0),BoundaryConditionOutlet(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),
Vin0(0),freqIn(0),t0_In(0),Vin_Type(0),Vout0(0),freqOut(0),t0_Out(0),Vout_Type(0),
TestPeriodic(0),TestPeriodicTime(0),TestPeriodicTimeConv(0),TestPeriodicSaveInterval(0),
comm(COMM)
{
if ( rank == 0 ) {
bool WriteHeader = !fileExists( "PoissonSolver_Convergence.csv" );
TIMELOG = fopen("PoissonSolver_Convergence.csv","a+");
if (WriteHeader)
fprintf(TIMELOG,"Timestep Error\n");
}
}
ScaLBL_Poisson::~ScaLBL_Poisson()
{
if ( TIMELOG )
fclose( TIMELOG );
}
void ScaLBL_Poisson::ReadParams(string filename){
@ -218,20 +234,19 @@ void ScaLBL_Poisson::ReadInput(){
void ScaLBL_Poisson::AssignSolidBoundary(double *poisson_solid)
{
size_t NLABELS=0;
signed char VALUE=0;
double AFFINITY=0.f;
auto LabelList = electric_db->getVector<int>( "SolidLabels" );
auto AffinityList = electric_db->getVector<double>( "SolidValues" );
NLABELS=LabelList.size();
size_t NLABELS = LabelList.size();
if (NLABELS != AffinityList.size()){
ERROR("Error: LB-Poisson Solver: SolidLabels and SolidValues must be the same length! \n");
}
double label_count[NLABELS];
double label_count_global[NLABELS];
std::vector<double> label_count( NLABELS, 0.0 );
std::vector<double> label_count_global( NLABELS, 0.0 );
// Assign the labels
for (size_t idx=0; idx<NLABELS; idx++) label_count[idx]=0;
@ -596,26 +611,11 @@ void ScaLBL_Poisson::Run(double *ChargeDensity, int timestep_from_Study){
}
void ScaLBL_Poisson::getConvergenceLog(int timestep,double error){
if (rank==0){
bool WriteHeader=false;
TIMELOG = fopen("PoissonSolver_Convergence.csv","r");
if (TIMELOG != NULL)
fclose(TIMELOG);
else
WriteHeader=true;
TIMELOG = fopen("PoissonSolver_Convergence.csv","a+");
if (WriteHeader)
{
fprintf(TIMELOG,"Timestep Error\n");
fprintf(TIMELOG,"%i %.5g\n",timestep,error);
fflush(TIMELOG);
}
else {
fprintf(TIMELOG,"%i %.5g\n",timestep,error);
fflush(TIMELOG);
}
void ScaLBL_Poisson::getConvergenceLog(int timestep,double error){
if ( rank == 0 ) {
fprintf(TIMELOG,"%i %.5g\n",timestep,error);
fflush(TIMELOG);
}
}

3
models/PoissonSolver.h
View File

@ -89,12 +89,13 @@ public:
private:
Utilities::MPI comm;
FILE *TIMELOG;
// filenames
char LocalRankString[8];
char LocalRankFilename[40];
char LocalRestartFile[40];
char OutputFilename[200];
FILE *TIMELOG;
//int rank,nprocs;
void LoadParams(std::shared_ptr<Database> db0);

33
sample_scripts/configure_spock_TPLs Executable file
View File

@ -0,0 +1,33 @@
export TPL_ROOT=/ccs/home/mbt/repos
export TPL_BUILDER=/ccs/home/mbt/repos/TPL-builder
export TPL_WEBPAGE=http://bitbucket.org/AdvancedMultiPhysics/tpl-builder/downloads
export INSTALL_DIR=/ccs/home/mbt/spock/install
module load cmake
module load llvm-amdgpu
module load hip
cmake \
-D CMAKE_BUILD_TYPE=Release \
-D CXX_STD=14 \
-D TPL_INSTALL_DIR:PATH="${INSTALL_DIR}" \
-D C_COMPILER=cc \
-D CXX_COMPILER=CC \
-D Fortran_COMPILER=ftn \
-D FFLAGS="-fPIC" \
-D CFLAGS="-fPIC" \
-D CXXFLAGS="-fPIC" \
-D LDFLAGS="" \
-D ENABLE_STATIC:BOOL=ON \
-D ENABLE_SHARED:BOOL=OFF \
-D PROCS_INSTALL=8 \
-D TPL_LIST:STRING="TIMER;ZLIB;HDF5;SILO" \
-D TIMER_URL="${TPL_ROOT}/TimerUtility" \
-D ZLIB_URL="http://zlib.net/zlib-1.2.11.tar.gz" \
-D HDF5_URL="${TPL_ROOT}/hdf5-1.8.12.tar.gz" \
-D BUILD_TYPE=x86_64 \
-D SILO_URL="${TPL_ROOT}/silo-4.10.2.tar.gz" \
${TPL_BUILDER}

30
sample_scripts/configure_spock_hip Executable file
View File

@ -0,0 +1,30 @@
module load cmake
module load llvm-amdgpu
module load hip
export TPL_DIR=/gpfs/alpine/stf006/proj-shared/mbt/spock/install
# configure
rm -rf CMake*
cmake \
-D CMAKE_BUILD_TYPE:STRING=Release \
-D CMAKE_C_COMPILER:PATH=cc \
-D CMAKE_CXX_COMPILER:PATH=CC \
-D CMAKE_CXX_STANDARD=14 \
-D USE_HIP=1 \
-D LINK_LIBRARIES=${HIP_PATH}/lib/libamdhip64.so \
-D USE_CUDA=0 \
-D CMAKE_CUDA_FLAGS="-arch sm_70 -Xptxas=-v -Xptxas -dlcm=cg -lineinfo" \
-D CMAKE_CUDA_HOST_COMPILER="gcc" \
-D USE_MPI=0 \
-D USE_HDF5=1 \
-D HDF5_DIRECTORY="${TPL_DIR}/hdf5" \
-D USE_SILO=0 \
-D SILO_DIRECTORY="${TPL_DIR}/silo" \
-D USE_DOXYGEN:BOOL=false \
-D USE_TIMER=0 \
~/repos/LBPM-WIA