xMerge branch 'master' into FOM_dev

This commit is contained in:
James McClure 2021-05-20 11:38:05 -04:00
commit ee34677826
14 changed files with 735 additions and 167 deletions

View File

@ -1,5 +1,8 @@
# Set some CMake properties # Set some CMake properties
CMAKE_MINIMUM_REQUIRED( VERSION 3.9 ) CMAKE_MINIMUM_REQUIRED( VERSION 3.9 )
if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.20.0")
CMAKE_POLICY( SET CMP0115 OLD )
endif()
MESSAGE("====================") MESSAGE("====================")

View File

@ -57,6 +57,6 @@ inline std::vector<std::string> splitList( const char *line, const char token )
} }
}; // namespace IO } // namespace IO
#endif #endif

View File

@ -18,7 +18,8 @@ typedef int DBfile;
#endif #endif
namespace IO::silo { namespace IO {
namespace silo {
enum FileMode { READ, WRITE, CREATE }; enum FileMode { READ, WRITE, CREATE };
@ -256,7 +257,9 @@ void writeMultiVar( DBfile *fid, const std::string &varname,
const std::vector<std::string> &subVarNames, const std::vector<int> &subVarTypes ); const std::vector<std::string> &subVarNames, const std::vector<int> &subVarTypes );
}; // namespace IO::silo } // namespace silo
} // namespace IO
#endif #endif
#include "IO/silo.hpp" #include "IO/silo.hpp"

View File

@ -13,7 +13,8 @@
#include <silo.h> #include <silo.h>
namespace IO::silo { namespace IO {
namespace silo {
/**************************************************** /****************************************************
@ -413,7 +414,8 @@ Array<TYPE> readTriMeshVariable( DBfile *fid, const std::string &varname )
} }
}; // namespace IO::silo } // namespace silo
} // namespace IO
#endif #endif

View File

@ -1,3 +1,9 @@
CMAKE_MINIMUM_REQUIRED(VERSION 3.10.0)
CMAKE_POLICY( SET CMP0057 NEW )
if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.20.0")
CMAKE_POLICY( SET CMP0115 OLD )
endif()
INCLUDE(CheckCCompilerFlag) INCLUDE(CheckCCompilerFlag)
INCLUDE(CheckCSourceCompiles) INCLUDE(CheckCSourceCompiles)
INCLUDE(CheckCXXCompilerFlag) INCLUDE(CheckCXXCompilerFlag)
@ -47,12 +53,24 @@ ELSEIF( NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION )
ENDIF() ENDIF()
# Check for gold linker
IF ( UNIX AND NOT APPLE AND NOT DISABLE_GOLD )
EXECUTE_PROCESS( COMMAND ${CMAKE_CXX_COMPILER} -fuse-ld=gold -Wl,--version ERROR_QUIET OUTPUT_VARIABLE LD_VERSION )
IF ( "${LD_VERSION}" MATCHES "GNU gold" )
MESSAGE( "Using gold linker" )
SET( GOLD_LINK_FLAGS "-fuse-ld=gold -Wl,--disable-new-dtags -Wl,--no-map-whole-files" )
SET( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GOLD_LINK_FLAGS}" )
SET( CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${GOLD_LINK_FLAGS}" )
ENDIF()
ENDIF()
# Add some default targets if they do not exist # Add some default targets if they do not exist
IF ( NOT TARGET copy-${PROJ}-Data ) IF ( NOT TARGET copy-${PROJ}-Data )
ADD_CUSTOM_TARGET( copy-${PROJ}-Data ALL ) ADD_CUSTOM_TARGET( copy-${PROJ}-Data ALL )
ENDIF() ENDIF()
IF ( NOT TARGET copy-${PROJ}-include ) IF ( NOT TARGET copy-${PROJ}-include )
ADD_CUSTOM_TARGET ( copy-${PROJ}-include ALL ) ADD_CUSTOM_TARGET( copy-${PROJ}-include ALL )
ENDIF() ENDIF()
@ -75,7 +93,7 @@ ENDMACRO()
MACRO( PRINT_ALL_VARIABLES ) MACRO( PRINT_ALL_VARIABLES )
GET_CMAKE_PROPERTY(_variableNames VARIABLES) GET_CMAKE_PROPERTY(_variableNames VARIABLES)
FOREACH(_variableName ${_variableNames}) FOREACH(_variableName ${_variableNames})
message(STATUS "${_variableName}=${${_variableName}}") MESSAGE(STATUS "${_variableName}=${${_variableName}}")
ENDFOREACH() ENDFOREACH()
ENDMACRO() ENDMACRO()
@ -108,28 +126,28 @@ MACRO( CONVERT_M4_FORTRAN IN LOCAL_PATH OUT_PATH )
IF ( NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/${OUT_PATH}" ) IF ( NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/${OUT_PATH}" )
FILE(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${OUT_PATH}" ) FILE(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${OUT_PATH}" )
ENDIF() ENDIF()
CONFIGURE_FILE ( ${IN} ${IN} COPYONLY ) CONFIGURE_FILE( ${IN} ${IN} COPYONLY )
IF ("${CMAKE_GENERATOR}" STREQUAL "Xcode") IF ("${CMAKE_GENERATOR}" STREQUAL "Xcode")
STRING(REGEX REPLACE ".F" ".o" OUT2 "${OUT}" ) STRING(REGEX REPLACE ".F" ".o" OUT2 "${OUT}" )
STRING(REGEX REPLACE ";" " " COMPILE_CMD "${CMAKE_Fortran_COMPILER} -c ${OUT} ${CMAKE_Fortran_FLAGS} -o ${OUT2}") STRING(REGEX REPLACE ";" " " COMPILE_CMD "${CMAKE_Fortran_COMPILER} -c ${OUT} ${CMAKE_Fortran_FLAGS} -o ${OUT2}")
STRING(REGEX REPLACE "\\\\" "" COMPILE_CMD "${COMPILE_CMD}") STRING(REGEX REPLACE "\\\\" "" COMPILE_CMD "${COMPILE_CMD}")
MESSAGE("COMPILE_CMD = ${COMPILE_CMD}") MESSAGE("COMPILE_CMD = ${COMPILE_CMD}")
SET( COMPILE_CMD ${COMPILE_CMD} ) SET( COMPILE_CMD ${COMPILE_CMD} )
add_custom_command( ADD_CUSTOM_COMMAND(
OUTPUT ${OUT2} OUTPUT ${OUT2}
COMMAND m4 -I${LOCAL_PATH} -I${SAMRAI_FORTDIR} ${M4DIRS} ${IN} > ${OUT} COMMAND m4 -I${LOCAL_PATH} -I${SAMRAI_FORTDIR} ${M4DIRS} ${IN} > ${OUT}
COMMAND ${COMPILE_CMD} COMMAND ${COMPILE_CMD}
DEPENDS ${IN} DEPENDS ${IN}
) )
set_source_files_properties(${OUT2} PROPERTIES GENERATED true) SET_SOURCE_FILES_PROPERTIES(${OUT2} PROPERTIES GENERATED true)
SET( SOURCES ${SOURCES} "${OUT2}" ) SET( SOURCES ${SOURCES} "${OUT2}" )
ELSE() ELSE()
add_custom_command( ADD_CUSTOM_COMMAND(
OUTPUT ${OUT} OUTPUT ${OUT}
COMMAND m4 -I${LOCAL_PATH} -I${SAMRAI_FORTDIR} ${M4DIRS} ${M4_OPTIONS} ${IN} > ${OUT} COMMAND m4 -I${LOCAL_PATH} -I${SAMRAI_FORTDIR} ${M4DIRS} ${M4_OPTIONS} ${IN} > ${OUT}
DEPENDS ${IN} DEPENDS ${IN}
) )
set_source_files_properties(${OUT} PROPERTIES GENERATED true) SET_SOURCE_FILES_PROPERTIES(${OUT} PROPERTIES GENERATED true)
SET( SOURCES ${SOURCES} "${OUT}" ) SET( SOURCES ${SOURCES} "${OUT}" )
ENDIF() ENDIF()
ENDMACRO() ENDMACRO()
@ -145,6 +163,10 @@ ENDMACRO()
# Add a project executable # Add a project executable
MACRO( ADD_${PROJ}_EXECUTABLE EXEFILE ) MACRO( ADD_${PROJ}_EXECUTABLE EXEFILE )
IF ( TARGET ${EXEFILE} AND NOT INCLUDE_${EXEFILE} )
MESSAGE( FATAL_ERROR "ADD_${PROJ}_EXECUTABLE should be called before adding tests" )
ENDIF()
SET( INCLUDE_${EXEFILE} TRUE )
ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} ) ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
INSTALL( TARGETS ${EXEFILE} DESTINATION ${${PROJ}_INSTALL_DIR}/bin ) INSTALL( TARGETS ${EXEFILE} DESTINATION ${${PROJ}_INSTALL_DIR}/bin )
ENDMACRO() ENDMACRO()
@ -164,7 +186,7 @@ ENDMACRO ()
# Find the source files # Find the source files
MACRO (FIND_FILES) MACRO( FIND_FILES )
# Find the C/C++ headers # Find the C/C++ headers
SET( T_HEADERS "" ) SET( T_HEADERS "" )
FILE( GLOB T_HEADERS "*.h" "*.H" "*.hh" "*.hpp" "*.I" ) FILE( GLOB T_HEADERS "*.h" "*.H" "*.hh" "*.hpp" "*.I" )
@ -193,7 +215,7 @@ MACRO (FIND_FILES)
SET( CSOURCES ${CSOURCES} ${T_CSOURCES} ) SET( CSOURCES ${CSOURCES} ${T_CSOURCES} )
SET( FSOURCES ${FSOURCES} ${T_FSOURCES} ) SET( FSOURCES ${FSOURCES} ${T_FSOURCES} )
SET( M4FSOURCES ${M4FSOURCES} ${T_M4FSOURCES} ) SET( M4FSOURCES ${M4FSOURCES} ${T_M4FSOURCES} )
SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${T_M4FSOURCES} ) SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${T_M4FSOURCES} ${CUDASOURCES} )
ENDMACRO() ENDMACRO()
@ -303,7 +325,7 @@ MACRO( INSTALL_${PROJ}_TARGET PACKAGE )
IF ( TARGET write_repo_version ) IF ( TARGET write_repo_version )
ADD_DEPENDENCIES( ${PACKAGE} write_repo_version ) ADD_DEPENDENCIES( ${PACKAGE} write_repo_version )
ENDIF() ENDIF()
ADD_DEPENDENCIES ( ${PACKAGE} copy-${PROJ}-include ) ADD_DEPENDENCIES( ${PACKAGE} copy-${PROJ}-include )
IF ( NOT ${PROJ}_LIB ) IF ( NOT ${PROJ}_LIB )
INSTALL( TARGETS ${PACKAGE} DESTINATION ${${PROJ}_INSTALL_DIR}/lib ) INSTALL( TARGETS ${PACKAGE} DESTINATION ${${PROJ}_INSTALL_DIR}/lib )
ENDIF() ENDIF()
@ -340,7 +362,7 @@ ENDMACRO()
# Macro to verify that a path has been set # Macro to verify that a path has been set
MACRO( VERIFY_PATH PATH_NAME ) MACRO( VERIFY_PATH PATH_NAME )
IF ("${PATH_NAME}" STREQUAL "") IF ("${PATH_NAME}" STREQUAL "")
MESSAGE ( FATAL_ERROR "Path is not set: ${PATH_NAME}" ) MESSAGE( FATAL_ERROR "Path is not set: ${PATH_NAME}" )
ENDIF() ENDIF()
IF ( NOT EXISTS "${PATH_NAME}" ) IF ( NOT EXISTS "${PATH_NAME}" )
MESSAGE( FATAL_ERROR "Path does not exist: ${PATH_NAME}" ) MESSAGE( FATAL_ERROR "Path does not exist: ${PATH_NAME}" )
@ -420,6 +442,9 @@ MACRO( IDENTIFY_COMPILER )
ELSEIF ( ${CMAKE_Fortran_COMPILER_ID} MATCHES "PGI") ELSEIF ( ${CMAKE_Fortran_COMPILER_ID} MATCHES "PGI")
SET(USING_PGF90 TRUE) SET(USING_PGF90 TRUE)
MESSAGE("Using pgf90") MESSAGE("Using pgf90")
ELSEIF( (${CMAKE_Fortran_COMPILER_ID} MATCHES "CRAY") OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "Cray") )
SET(USING_CRAY TRUE)
MESSAGE("Using Cray")
ELSEIF ( (${CMAKE_Fortran_COMPILER_ID} MATCHES "CLANG") OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "Clang") OR ELSEIF ( (${CMAKE_Fortran_COMPILER_ID} MATCHES "CLANG") OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "Clang") OR
(${CMAKE_Fortran_COMPILER_ID} MATCHES "FLANG") OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "Flang") ) (${CMAKE_Fortran_COMPILER_ID} MATCHES "FLANG") OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "Flang") )
SET(USING_FLANG TRUE) SET(USING_FLANG TRUE)
@ -440,7 +465,7 @@ MACRO( SET_WARNINGS )
# Note: adding -Wlogical-op causes a wierd linking error on Titan using the nvcc wrapper: # Note: adding -Wlogical-op causes a wierd linking error on Titan using the nvcc wrapper:
# /usr/bin/ld: cannot find gical-op: No such file or directory # /usr/bin/ld: cannot find gical-op: No such file or directory
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Woverloaded-virtual -Wsign-compare") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -Woverloaded-virtual -Wsign-compare -Wformat-security")
ELSEIF ( USING_MSVC ) ELSEIF ( USING_MSVC )
# Add Microsoft specifc compiler options # Add Microsoft specifc compiler options
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /D _SCL_SECURE_NO_WARNINGS /D _CRT_SECURE_NO_WARNINGS /D _ITERATOR_DEBUG_LEVEL=0 /wd4267" ) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /D _SCL_SECURE_NO_WARNINGS /D _CRT_SECURE_NO_WARNINGS /D _ITERATOR_DEBUG_LEVEL=0 /wd4267" )
@ -448,7 +473,7 @@ MACRO( SET_WARNINGS )
ELSEIF ( USING_ICC ) ELSEIF ( USING_ICC )
# Add Intel specifc compiler options # Add Intel specifc compiler options
SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -Wall" ) SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -Wall" )
SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall" ) SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall -wd1011" )
ELSEIF ( USING_CRAY ) ELSEIF ( USING_CRAY )
# Add default compiler options # Add default compiler options
SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS}") SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS}")
@ -461,8 +486,8 @@ MACRO( SET_WARNINGS )
SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} --diag_suppress 111,128,185") SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} --diag_suppress 111,128,185")
ELSEIF ( USING_CLANG ) ELSEIF ( USING_CLANG )
# Add default compiler options # Add default compiler options
SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -Wall") SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -Wall -Wextra")
SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall -Wno-missing-braces -Wmissing-field-initializers -ftemplate-depth=1024") SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic -Wno-missing-braces -Wmissing-field-initializers -ftemplate-depth=1024")
ELSEIF ( USING_XL ) ELSEIF ( USING_XL )
# Add default compiler options # Add default compiler options
SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -Wall") SET(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -Wall")
@ -651,7 +676,7 @@ MACRO( COPY_MESH_FILE MESHNAME )
ENDFOREACH() ENDFOREACH()
# We have either found the mesh or failed # We have either found the mesh or failed
IF ( NOT MESHPATH ) IF ( NOT MESHPATH )
MESSAGE ( WARNING "Cannot find mesh: " ${MESHNAME} ) MESSAGE( WARNING "Cannot find mesh: " ${MESHNAME} )
ELSE () ELSE ()
SET( MESHPATH2 ) SET( MESHPATH2 )
FOREACH( tmp ${MESHPATH} ) FOREACH( tmp ${MESHPATH} )
@ -666,25 +691,16 @@ ENDMACRO()
# Link the libraries to the given target # Link the libraries to the given target
MACRO( TARGET_LINK_EXTERNAL_LIBRARIES TARGET_NAME ) MACRO( TARGET_LINK_EXTERNAL_LIBRARIES TARGET_NAME )
FOREACH ( tmp ${TPL_LIBS} ) # Include external libraries
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
FOREACH ( tmp ${EXTERNAL_LIBS} ) FOREACH ( tmp ${EXTERNAL_LIBS} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} ) TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH() ENDFOREACH()
FOREACH ( tmp ${LAPACK_LIBS} ) # Include libraries found through the TPL builder
FOREACH ( tmp ${TPL_LIBS} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} ) TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH() ENDFOREACH()
FOREACH ( tmp ${BLAS_LIBS} ) # Include CMake implicit libraries
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} ) FOREACH ( tmp ${CMAKE_CUDA_IMPLICIT_LINK_LIBRARIES} ${CMAKE_C_IMPLICIT_LINK_LIBRARIES}
ENDFOREACH()
FOREACH ( tmp ${BLAS_LAPACK_LIBS} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
FOREACH ( tmp ${MPI_LIBRARIES} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
FOREACH ( tmp ${CMAKE_C_IMPLICIT_LINK_LIBRARIES}
${CMAKE_CXX_IMPLICIT_LINK_LIBRARIES} ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES} ) ${CMAKE_CXX_IMPLICIT_LINK_LIBRARIES} ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} ) TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH() ENDFOREACH()
@ -715,8 +731,8 @@ ENDFUNCTION()
MACRO( ADD_PROJ_EXE_DEP EXE ) MACRO( ADD_PROJ_EXE_DEP EXE )
# Add the package dependencies # Add the package dependencies
IF( ${PROJ}_TEST_LIB_EXISTS ) IF( ${PROJ}_TEST_LIB_EXISTS )
ADD_DEPENDENCIES ( ${EXE} ${PACKAGE_TEST_LIB} ) ADD_DEPENDENCIES( ${EXE} ${PACKAGE_TEST_LIB} )
TARGET_LINK_LIBRARIES ( ${EXE} ${PACKAGE_TEST_LIB} ) TARGET_LINK_LIBRARIES( ${EXE} ${PACKAGE_TEST_LIB} )
ENDIF() ENDIF()
# Add the executable to the dependencies of check and build-test # Add the executable to the dependencies of check and build-test
ADD_DEPENDENCIES( check ${EXE} ) ADD_DEPENDENCIES( check ${EXE} )
@ -783,7 +799,7 @@ FUNCTION( ADD_PROJ_PROVISIONAL_TEST EXEFILE )
SET( CXXFILE ${EXEFILE} ) SET( CXXFILE ${EXEFILE} )
SET( TESTS_SO_FAR ${TESTS_SO_FAR} ${EXEFILE} ) SET( TESTS_SO_FAR ${TESTS_SO_FAR} ${EXEFILE} )
# Check if we want to add the test to all # Check if we want to add the test to all
IF ( NOT EXCLUDE_TESTS_FROM_ALL ) IF ( NOT EXCLUDE_TESTS_FROM_ALL OR INCLUDE_${EXEFILE} )
ADD_EXECUTABLE( ${EXEFILE} ${CXXFILE} ) ADD_EXECUTABLE( ${EXEFILE} ${CXXFILE} )
ELSE() ELSE()
ADD_EXECUTABLE( ${EXEFILE} EXCLUDE_FROM_ALL ${CXXFILE} ) ADD_EXECUTABLE( ${EXEFILE} EXCLUDE_FROM_ALL ${CXXFILE} )
@ -818,123 +834,218 @@ MACRO( CREATE_TEST_NAME TEST ${ARGN} )
FOREACH( tmp ${ARGN}) FOREACH( tmp ${ARGN})
SET( TESTNAME "${TESTNAME}--${tmp}") SET( TESTNAME "${TESTNAME}--${tmp}")
endforeach() endforeach()
# STRING(REGEX REPLACE "--" "-" TESTNAME ${TESTNAME} )
SET( LAST_TESTNAME ${TESTNAME} PARENT_SCOPE ) SET( LAST_TESTNAME ${TESTNAME} PARENT_SCOPE )
ENDMACRO() ENDMACRO()
# Function to add the resource locks to an executable # Function to add the resource locks to an executable
FUNCTION( ADD_RESOURCE_LOCK TESTNAME EXEFILE ${ARGN} ) FUNCTION( ADD_RESOURCE TESTNAME RESOURCE )
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES RESOURCE_LOCK ${RESOURCE} )
ENDFUNCTION()
FUNCTION( ADD_RESOURCE_LOCKS TESTNAME EXEFILE ${ARGN} )
IF ( NOT DISABLE_RESOURCE_LOCK ) IF ( NOT DISABLE_RESOURCE_LOCK )
IF ( NOT ARGN ) IF ( NOT ARGN )
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES RESOURCE_LOCK ${EXEFILE} ) ADD_RESOURCE( ${TESTNAME} ${EXEFILE} )
ELSE() ELSE()
FOREACH( tmp ${ARGN} ) FOREACH( tmp ${ARGN} )
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES RESOURCE_LOCK ${tmp} ) ADD_RESOURCE( ${TESTNAME} ${tmp} )
ENDFOREACH() ENDFOREACH()
ENDIF() ENDIF()
ENDIF() ENDIF()
ENDFUNCTION() ENDFUNCTION()
# Add a executable as a test # Function to add a test
FUNCTION( ADD_${PROJ}_TEST EXEFILE ${ARGN} ) FUNCTION( CALL_ADD_TEST EXEFILE PROCS THREADS WEEKLY TESTBUILDER ${ARGN} )
# Check if we actually want to add the test # Check if we actually want to add the test
KEEP_TEST( RESULT ) KEEP_TEST( RESULT )
IF ( NOT RESULT ) IF ( NOT RESULT )
MESSAGE( "Discarding test: ${EXEFILE}" )
RETURN() RETURN()
ENDIF() ENDIF()
# Add the provisional test # Check if we are dealing with a TestBuilder test
ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} ) IF ( ${TESTBUILDER} )
CREATE_TEST_NAME( ${EXEFILE} ${ARGN} ) # Add the test to the TestBuilder sources
IF ( USE_MPI_FOR_SERIAL_TESTS ) SET( TESTBUILDER_SOURCES ${TESTBUILDER_SOURCES} ${EXEFILE} PARENT_SCOPE )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
ELSE() ELSE()
ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} ) # Add the provisional test
ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
ENDIF() ENDIF()
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS 1 ) # Create the test name
ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} ) SET( TESTNAME ${EXEFILE} )
ENDFUNCTION() #IF ( ${TESTBUILDER} )
# SET( TESTNAME TB_${TESTNAME} )
#ENDIF()
# Add a executable as a weekly test IF ( "${PROCS}" GREATER "1")
FUNCTION( ADD_${PROJ}_WEEKLY_TEST EXEFILE PROCS ${ARGN} ) SET( TESTNAME ${TESTNAME}_${PROCS}procs )
# Check if we actually want to add the test
KEEP_TEST( RESULT )
IF ( NOT RESULT )
RETURN()
ENDIF() ENDIF()
# Add the provisional test IF ( "${THREADS}" GREATER "1")
ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} ) SET( TESTNAME ${TESTNAME}_${THREADS}threads )
IF( ${PROCS} STREQUAL "1" )
CREATE_TEST_NAME( "${EXEFILE}_WEEKLY" ${ARGN} )
ELSEIF( (USE_MPI OR USE_EXT_MPI) AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) )
CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_WEEKLY" ${ARGN} )
ENDIF() ENDIF()
IF ( ${PROCS} GREATER ${TEST_MAX_PROCS} ) IF ( ${WEEKLY} )
SET( TESTNAME ${TESTNAME}_WEEKLY )
ENDIF()
CREATE_TEST_NAME( ${TESTNAME} ${ARGN} )
# Create the tests for ctest
MATH( EXPR TOT_PROCS "${PROCS} * ${THREADS}" )
IF ( ${TOT_PROCS} EQUAL 0 )
# Skip test (provisional)
ELSEIF ( ${TOT_PROCS} GREATER ${TEST_MAX_PROCS} )
MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})") MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})")
ELSEIF( ${PROCS} STREQUAL "1" ) ELSEIF ( ( ${PROCS} STREQUAL "1" ) AND NOT USE_MPI_FOR_SERIAL_TESTS )
CREATE_TEST_NAME( "${EXEFILE}_WEEKLY" ${ARGN} ) IF ( ${TESTBUILDER} )
IF ( USE_MPI_FOR_SERIAL_TESTS ) ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${TB_TARGET}> ${EXEFILE} ${ARGN} )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
ELSE() ELSE()
ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} ) ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
ENDIF() ENDIF()
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS 1 ) SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} )
ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} ) ADD_RESOURCE_LOCKS( ${TESTNAME} ${EXEFILE} ${ARGN} )
ELSEIF( (USE_MPI OR USE_EXT_MPI) AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) ) ELSEIF ( USE_MPI OR USE_EXT_MPI )
CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_WEEKLY" ${ARGN} ) IF ( ${TESTBUILDER} )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} ) ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${TB_TARGET}> ${EXEFILE} ${ARGN} )
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} ) ELSE()
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
ENDIF()
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none ) SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} ) SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} )
ADD_RESOURCE_LOCKS( ${TESTNAME} ${EXEFILE} ${ARGN} )
ENDIF() ENDIF()
ENDFUNCTION() ENDFUNCTION()
# Add a executable as a parallel test # Functions to create a test builder
FUNCTION( ADD_${PROJ}_TEST_PARALLEL EXEFILE PROCS ${ARGN} ) FUNCTION( INITIALIZE_TESTBUILDER )
SET( TESTBUILDER_SOURCES PARENT_SCOPE )
# Check if we actually want to add the test # Check if we actually want to add the test
KEEP_TEST( RESULT ) KEEP_TEST( RESULT )
IF ( NOT RESULT ) IF ( NOT RESULT )
RETURN() RETURN()
ENDIF() ENDIF()
# Add the provisional test # Check if the target does not exist (may not be added yet or we are re-configuring)
ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} ) STRING(REGEX REPLACE "${${PROJ}_SOURCE_DIR}/" "" TB_TARGET "TB-${PROJ}-${CMAKE_CURRENT_SOURCE_DIR}" )
CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs" ${ARGN} ) STRING(REGEX REPLACE "/" "-" TB_TARGET ${TB_TARGET} )
IF ( NOT ( USE_MPI OR USE_EXT_MPI ) ) IF ( NOT TARGET ${TB_TARGET} )
MESSAGE("Disabling test ${TESTNAME} (configured without MPI)") GLOBAL_SET( ${TB_TARGET}-BINDIR )
ELSEIF ( ${PROCS} GREATER ${TEST_MAX_PROCS} )
MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})")
ELSE()
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )
ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
ENDIF() ENDIF()
# Check if test has already been added
IF ( NOT ${TB_TARGET}-BINDIR )
GLOBAL_SET( ${TB_TARGET}-BINDIR "${CMAKE_CURRENT_BINARY_DIR}" )
# Create the initial file
IF ( NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" )
FILE( WRITE "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" "// Auto generated file\n\n" )
ENDIF()
# The target has not been added
SET( CXXFILE "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" )
SET( TESTS_SO_FAR ${TESTS_SO_FAR} ${TB_TARGET} )
# Check if we want to add the test to all
IF ( NOT EXCLUDE_TESTS_FROM_ALL )
ADD_EXECUTABLE( ${TB_TARGET} ${CXXFILE} )
ELSE()
ADD_EXECUTABLE( ${TB_TARGET} EXCLUDE_FROM_ALL ${CXXFILE} )
ENDIF()
SET_TARGET_PROPERTIES( ${TB_TARGET} PROPERTIES OUTPUT_NAME "TB" )
ELSEIF ( NOT ${TB-BINDIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR} )
# We are trying to add 2 different tests with the same name
MESSAGE( "Existing test: ${TB-BINDIR}/${TB_TARGET}" )
MESSAGE( "New test: ${CMAKE_CURRENT_BINARY_DIR}/${TB_TARGET}}" )
MESSAGE( FATAL_ERROR "Trying to add 2 different tests with the same name" )
ENDIF()
SET( TB_TARGET ${TB_TARGET} PARENT_SCOPE )
ENDFUNCTION()
FUNCTION( FINALIZE_TESTBUILDER )
# Check if we actually want to add the test
KEEP_TEST( RESULT )
IF ( NOT RESULT )
RETURN()
ENDIF()
# Create the library for the test builder
IF ( TESTBUILDER_SOURCES )
LIST( REMOVE_DUPLICATES TESTBUILDER_SOURCES )
SET( TB_TARGET_SOURCES "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" )
FOREACH ( tmp ${TESTBUILDER_SOURCES} )
GET_FILENAME_COMPONENT( tmp ${tmp} ABSOLUTE )
SET( TB_TARGET_SOURCES ${TB_TARGET_SOURCES} ${tmp} )
ENDFOREACH()
TARGET_SOURCES( ${TB_TARGET} PUBLIC ${TB_TARGET_SOURCES} )
ADD_DEPENDENCIES( ${TB_TARGET} copy-${PROJ}-include )
ENDIF()
# Add the dependencies to the test builder
ADD_PROJ_EXE_DEP( ${TB_TARGET} )
IF ( CURRENT_LIBRARY )
IF ( NOT TARGET ${CURRENT_LIBRARY}-test )
ADD_CUSTOM_TARGET( ${CURRENT_LIBRARY}-test )
ENDIF()
ADD_DEPENDENCIES( ${CURRENT_LIBRARY}-test ${TB_TARGET} )
ENDIF()
SET( TB_TARGET ${TB_TARGET} PARENT_SCOPE )
# Finish generating TestBuilder.cpp
SET( TB_FILE "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder-tmp.cpp" )
FILE( WRITE "${TB_FILE}" "// Auto generated file\n\n" )
FILE( APPEND "${TB_FILE}" "#include <cstring>\n" )
FILE( APPEND "${TB_FILE}" "#include <iostream>\n\n" )
FOREACH( tmp ${TESTBUILDER_SOURCES} )
FILE( APPEND "${TB_FILE}" "extern int ${tmp}( int argc, char *argv[] );\n" )
ENDFOREACH()
FILE( APPEND "${TB_FILE}" "\n\n// Main Program\n" )
FILE( APPEND "${TB_FILE}" "int main( int argc, char *argv[] )\n" )
FILE( APPEND "${TB_FILE}" "{\n" )
FILE( APPEND "${TB_FILE}" " if ( argc < 2 ) {\n" )
FILE( APPEND "${TB_FILE}" " std::cerr << \"Invaild number of arguments for TestBuilder\";\n " )
FILE( APPEND "${TB_FILE}" " return 1;\n" )
FILE( APPEND "${TB_FILE}" " }\n\n" )
FILE( APPEND "${TB_FILE}" " int rtn = 0;\n" )
FILE( APPEND "${TB_FILE}" " if ( strcmp( argv[1], \"NULL\" ) == 0 ) {\n" )
FOREACH( tmp ${TESTBUILDER_SOURCES} )
FILE( APPEND "${TB_FILE}" " } else if ( strcmp( argv[1], \"${tmp}\" ) == 0 ) {\n" )
FILE( APPEND "${TB_FILE}" " rtn = ${tmp}( argc-1, &argv[1] );\n" )
ENDFOREACH()
FILE( APPEND "${TB_FILE}" " } else {\n" )
FILE( APPEND "${TB_FILE}" " std::cerr << \"Unknown test\";\n" )
FILE( APPEND "${TB_FILE}" " return 1;\n" )
FILE( APPEND "${TB_FILE}" " }\n\n" )
FILE( APPEND "${TB_FILE}" " return rtn;\n" )
FILE( APPEND "${TB_FILE}" "}\n" )
EXECUTE_PROCESS( COMMAND ${CMAKE_COMMAND} -E copy_if_different "${TB_FILE}" "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" )
ENDFUNCTION() ENDFUNCTION()
# Add a parallel test that may use both MPI and threads # Convience functions to add a test
# This allows us to correctly compute the number of processors used by the test MACRO( ADD_${PROJ}_TEST EXEFILE ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE FALSE ${ARGN} )
ENDMACRO()
MACRO( ADD_${PROJ}_TEST_PARALLEL EXEFILE PROCS ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 FALSE FALSE ${ARGN} )
ENDMACRO()
MACRO( ADD_${PROJ}_WEEKLY_TEST EXEFILE PROCS ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 TRUE FALSE ${ARGN} )
ENDMACRO()
MACRO( ADD_${PROJ}_TEST_THREAD_MPI EXEFILE PROCS THREADS ${ARGN} ) MACRO( ADD_${PROJ}_TEST_THREAD_MPI EXEFILE PROCS THREADS ${ARGN} )
ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} ) CALL_ADD_TEST( ${EXEFILE} ${PROCS} ${THREADS} FALSE FALSE ${ARGN} )
CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_${THREADS}threads" ${ARGN} ) ENDMACRO()
MATH( EXPR TOT_PROCS "${PROCS} * ${THREADS}" ) MACRO( ADD_${PROJ}_TEST_1_2_4 EXEFILE ${ARGN} )
IF ( ${TOT_PROCS} GREATER ${TEST_MAX_PROCS} ) CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE FALSE ${ARGN} )
MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})") CALL_ADD_TEST( ${EXEFILE} 2 1 FALSE FALSE ${ARGN} )
ELSEIF ( ( ${PROCS} STREQUAL "1" ) AND NOT USE_MPI_FOR_SERIAL_TESTS ) CALL_ADD_TEST( ${EXEFILE} 4 1 FALSE FALSE ${ARGN} )
ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} ) ENDMACRO()
SET_TESTS_PROPERTIES ( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} ) MACRO( ADD_TB_PROVISIONAL_TEST EXEFILE ${ARGN} )
ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} ) CALL_ADD_TEST( ${EXEFILE} 0 1 FALSE TRUE ${ARGN} )
ELSEIF ( USE_MPI OR USE_EXT_MPI ) ENDMACRO()
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} ) MACRO( ADD_TB_TEST EXEFILE ${ARGN} )
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none ) CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE TRUE ${ARGN} )
SET_TESTS_PROPERTIES ( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} ) ENDMACRO()
ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} ) MACRO( ADD_TB_TEST_PARALLEL EXEFILE PROCS ${ARGN} )
ENDIF() CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 FALSE TRUE ${ARGN} )
ENDMACRO()
MACRO( ADD_TB_WEEKLY_TEST EXEFILE PROCS ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 TRUE TRUE ${ARGN} )
ENDMACRO()
MACRO( ADD_TB_TEST_THREAD_MPI EXEFILE PROCS THREADS ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} ${PROCS} ${THREADS} FALSE TRUE ${ARGN} )
ENDMACRO()
MACRO( ADD_TB_TEST_1_2_4 EXEFILE ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE TRUE ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} 2 1 FALSE TRUE ${ARGN} )
CALL_ADD_TEST( ${EXEFILE} 4 1 FALSE TRUE ${ARGN} )
ENDMACRO() ENDMACRO()
@ -966,7 +1077,7 @@ FUNCTION( ADD_${PROJ}_EXAMPLE EXEFILE PROCS ${ARGN} )
ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} ) ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
ELSEIF ( USE_EXT_MPI AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) ) ELSEIF ( USE_EXT_MPI AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) )
CREATE_TEST_NAME( "example--${EXEFILE}_${PROCS}procs" ${ARGN} ) CREATE_TEST_NAME( "example--${EXEFILE}_${PROCS}procs" ${ARGN} )
ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} ) ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none ) SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
ENDIF() ENDIF()
SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} ) SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )
@ -1038,6 +1149,8 @@ FUNCTION( ADD_MATLAB_MEX SOURCE )
MATLAB_ADD_MEX( MATLAB_ADD_MEX(
NAME ${TARGET} NAME ${TARGET}
SRC ${SOURCE} SRC ${SOURCE}
MODULE
R2017b
) )
TARGET_LINK_LIBRARIES( ${TARGET} ${MATLAB_TARGET} ) TARGET_LINK_LIBRARIES( ${TARGET} ${MATLAB_TARGET} )
ADD_PROJ_EXE_DEP( ${TARGET} ) ADD_PROJ_EXE_DEP( ${TARGET} )
@ -1074,9 +1187,11 @@ ENDMACRO()
# Create a script to start matlab preloading libraries # Create a script to start matlab preloading libraries
FUNCTION( CREATE_MATLAB_WRAPPER ) FUNCTION( CREATE_MATLAB_WRAPPER )
SET( MATLABPATH ${MATLABPATH} "${${PROJ}_INSTALL_DIR}/mex" )
SET( tmp_libs ${MEX_LIBCXX} ${MEX_FILES} ) SET( tmp_libs ${MEX_LIBCXX} ${MEX_FILES} )
STRING(REGEX REPLACE ";" ":" tmp_libs "${tmp_libs}") LIST( REMOVE_DUPLICATES MATLABPATH )
STRING(REGEX REPLACE ";" ":" tmp_path "${MATLABPATH}") STRING( REGEX REPLACE ";" ":" tmp_libs "${tmp_libs}" )
STRING( REGEX REPLACE ";" ":" tmp_path "${MATLABPATH}" )
IF ( USING_MSVC ) IF ( USING_MSVC )
# Create a matlab wrapper for windows # Create a matlab wrapper for windows
SET( MATLAB_GUI "${CMAKE_CURRENT_BINARY_DIR}/tmp/matlab-gui.bat" ) SET( MATLAB_GUI "${CMAKE_CURRENT_BINARY_DIR}/tmp/matlab-gui.bat" )
@ -1185,7 +1300,7 @@ MACRO( ADD_DISTCLEAN ${ARGN} )
compile_commands.json compile_commands.json
${ARGN} ${ARGN}
) )
ADD_CUSTOM_TARGET (distclean @echo cleaning for source distribution) ADD_CUSTOM_TARGET(distclean @echo cleaning for source distribution)
IF (UNIX) IF (UNIX)
ADD_CUSTOM_COMMAND( ADD_CUSTOM_COMMAND(
DEPENDS clean DEPENDS clean
@ -1269,3 +1384,4 @@ FUNCTION( FIND_PYTHON_MODULE MODULE)
ENDIF() ENDIF()
SET( PY_${MODULE2}_FOUND ${PY_${MODULE2}_FOUND} PARENT_SCOPE ) SET( PY_${MODULE2}_FOUND ${PY_${MODULE2}_FOUND} PARENT_SCOPE )
ENDFUNCTION(FIND_PYTHON_MODULE) ENDFUNCTION(FIND_PYTHON_MODULE)

View File

@ -595,7 +595,15 @@ template<class TYPE>
void MPI_CLASS::sendrecv( const TYPE *sendbuf, int sendcount, int dest, int sendtag, void MPI_CLASS::sendrecv( const TYPE *sendbuf, int sendcount, int dest, int sendtag,
TYPE *recvbuf, int recvcount, int source, int recvtag ) const TYPE *recvbuf, int recvcount, int source, int recvtag ) const
{ {
ERROR( "Not implimented" ); if ( getSize() == 1 ) {
ASSERT( dest == 0 );
ASSERT( source == 0 );
ASSERT( sendcount == recvcount );
ASSERT( sendtag == recvtag );
memcpy( recvbuf, sendbuf, sendcount * sizeof( TYPE ) );
} else {
ERROR( "Not implimented for " + std::string( typeid( TYPE ).name() ) );
}
} }
#else #else
template<class TYPE> template<class TYPE>

View File

@ -1,6 +1,7 @@
#include <math.h> #include <math.h>
#include <stdio.h> #include <stdio.h>
#include <cuda_profiler_api.h> #include "hip/hip_runtime.h"
#include "hip/hip_runtime_api.h"
#define STOKES #define STOKES
@ -3395,7 +3396,8 @@ extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, doubl
extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel, extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np) double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np)
{ {
cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField, cudaFuncCachePreferL1);
hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField, hipFuncCachePreferL1);
dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField<<<NBLOCKS,NTHREADS >>>(neighborList, Map, hq, Den, Phi, ColorGrad, Vel, dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField<<<NBLOCKS,NTHREADS >>>(neighborList, Map, hq, Den, Phi, ColorGrad, Vel,
rhoA, rhoB, tauM, W, start, finish, Np); rhoA, rhoB, tauM, W, start, finish, Np);
hipError_t err = hipGetLastError(); hipError_t err = hipGetLastError();
@ -3407,7 +3409,8 @@ extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map
extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel, extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){ double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){
cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField, cudaFuncCachePreferL1); hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField, hipFuncCachePreferL1);

dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField<<<NBLOCKS,NTHREADS >>>( Map, hq, Den, Phi, ColorGrad, Vel, rhoA, rhoB, tauM, W, start, finish, Np); dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField<<<NBLOCKS,NTHREADS >>>( Map, hq, Den, Phi, ColorGrad, Vel, rhoA, rhoB, tauM, W, start, finish, Np);
hipError_t err = hipGetLastError(); hipError_t err = hipGetLastError();
if (hipSuccess != err){ if (hipSuccess != err){
@ -3418,7 +3421,7 @@ extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, dou
extern "C" void ScaLBL_D3Q7_ComputePhaseField(int *Map, double *hq, double *Den, double *Phi, double rhoA, double rhoB, int start, int finish, int Np){ extern "C" void ScaLBL_D3Q7_ComputePhaseField(int *Map, double *hq, double *Den, double *Phi, double rhoA, double rhoB, int start, int finish, int Np){
cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q7_ComputePhaseField, cudaFuncCachePreferL1); hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q7_ComputePhaseField, hipFuncCachePreferL1);
dvc_ScaLBL_D3Q7_ComputePhaseField<<<NBLOCKS,NTHREADS >>>( Map, hq, Den, Phi, rhoA, rhoB, start, finish, Np); dvc_ScaLBL_D3Q7_ComputePhaseField<<<NBLOCKS,NTHREADS >>>( Map, hq, Den, Phi, rhoA, rhoB, start, finish, Np);
hipError_t err = hipGetLastError(); hipError_t err = hipGetLastError();
if (hipSuccess != err){ if (hipSuccess != err){
@ -3431,7 +3434,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, dou
double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz,
int strideY, int strideZ, int start, int finish, int Np){ int strideY, int strideZ, int start, int finish, int Np){
cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel, cudaFuncCachePreferL1); hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel, hipFuncCachePreferL1);
dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel<<<NBLOCKS,NTHREADS >>>(neighborList, Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad, dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel<<<NBLOCKS,NTHREADS >>>(neighborList, Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad,
rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np);
hipError_t err = hipGetLastError(); hipError_t err = hipGetLastError();
@ -3444,7 +3447,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double
double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz,
int strideY, int strideZ, int start, int finish, int Np){ int strideY, int strideZ, int start, int finish, int Np){
cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel, cudaFuncCachePreferL1); hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel, hipFuncCachePreferL1);
dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel<<<NBLOCKS,NTHREADS >>>(Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad, dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel<<<NBLOCKS,NTHREADS >>>(Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad,
rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np);
hipError_t err = hipGetLastError(); hipError_t err = hipGetLastError();
@ -3502,7 +3505,7 @@ extern "C" void ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(int *Map, double *hq,
extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborList, double *dist, double *Vel, double *Pressure, extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborList, double *dist, double *Vel, double *Pressure,
double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){ double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){
cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK, cudaFuncCachePreferL1); hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK, hipFuncCachePreferL1);
dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK<<<NBLOCKS,NTHREADS >>>(neighborList, dist, Vel, Pressure, dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK<<<NBLOCKS,NTHREADS >>>(neighborList, dist, Vel, Pressure,
tau, rho0, Fx, Fy, Fz, start, finish, Np); tau, rho0, Fx, Fy, Fz, start, finish, Np);
hipError_t err = hipGetLastError(); hipError_t err = hipGetLastError();
@ -3514,7 +3517,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborLis
extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, double *Vel, double *Pressure, extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, double *Vel, double *Pressure,
double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){ double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){
cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK, cudaFuncCachePreferL1);
hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK, hipFuncCachePreferL1);
dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK<<<NBLOCKS,NTHREADS >>>(dist, Vel, Pressure, dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK<<<NBLOCKS,NTHREADS >>>(dist, Vel, Pressure,
tau, rho0, Fx, Fy, Fz, start, finish, Np); tau, rho0, Fx, Fy, Fz, start, finish, Np);
hipError_t err = hipGetLastError(); hipError_t err = hipGetLastError();
@ -3525,4 +3529,4 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, d
extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad,int strideY, int strideZ, int start, int finish, int Np){ extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad,int strideY, int strideZ, int start, int finish, int Np){
} }

View File

@ -169,6 +169,13 @@ void ScaLBL_ColorModel::ReadParams(string filename){
} }
domain_db->putScalar<int>( "BC", BoundaryCondition ); domain_db->putScalar<int>( "BC", BoundaryCondition );
} }
else if (protocol == "fractional flow"){
if (BoundaryCondition != 0 && BoundaryCondition != 5){
BoundaryCondition = 0;
if (rank==0) printf("WARNING: protocol (fractional flow) supports only full periodic boundary condition \n");
}
domain_db->putScalar<int>( "BC", BoundaryCondition );
}
} }
void ScaLBL_ColorModel::SetDomain(){ void ScaLBL_ColorModel::SetDomain(){
@ -533,14 +540,39 @@ void ScaLBL_ColorModel::Initialize(){
double ScaLBL_ColorModel::Run(int returntime){ double ScaLBL_ColorModel::Run(int returntime){
int nprocs=nprocx*nprocy*nprocz; int nprocs=nprocx*nprocy*nprocz;
const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
//************ MAIN ITERATION LOOP ***************************************/ //************ MAIN ITERATION LOOP ***************************************/
comm.barrier(); comm.barrier();
PROFILE_START("Loop"); PROFILE_START("Loop");
//std::shared_ptr<Database> analysis_db; //std::shared_ptr<Database> analysis_db;
bool Regular = false; bool Regular = false;
bool RESCALE_FORCE = false;
bool SET_CAPILLARY_NUMBER = false;
double tolerance = 0.01;
auto current_db = db->cloneDatabase(); auto current_db = db->cloneDatabase();
auto flow_db = db->getDatabase( "FlowAdaptor" );
int MIN_STEADY_TIMESTEPS = flow_db->getWithDefault<int>( "min_steady_timesteps", 1000000 );
int MAX_STEADY_TIMESTEPS = flow_db->getWithDefault<int>( "max_steady_timesteps", 1000000 );
int RESCALE_FORCE_AFTER_TIMESTEP = MAX_STEADY_TIMESTEPS*2;
double capillary_number = 1.0e-5;
double Ca_previous = 0.0;
if (color_db->keyExists( "capillary_number" )){
capillary_number = color_db->getScalar<double>( "capillary_number" );
SET_CAPILLARY_NUMBER=true;
}
if (color_db->keyExists( "rescale_force_after_timestep" )){
RESCALE_FORCE_AFTER_TIMESTEP = color_db->getScalar<int>( "rescale_force_after_timestep" );
RESCALE_FORCE = true;
}
if (analysis_db->keyExists( "tolerance" )){
tolerance = analysis_db->getScalar<double>( "tolerance" );
}
runAnalysis analysis( current_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map );
auto t1 = std::chrono::system_clock::now(); auto t1 = std::chrono::system_clock::now();
int CURRENT_TIMESTEP = 0;
int START_TIMESTEP = timestep; int START_TIMESTEP = timestep;
int EXIT_TIMESTEP = min(timestepMax,returntime); int EXIT_TIMESTEP = min(timestepMax,returntime);
while (timestep < EXIT_TIMESTEP ) { while (timestep < EXIT_TIMESTEP ) {
@ -626,7 +658,158 @@ double ScaLBL_ColorModel::Run(int returntime){
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_Comm->Barrier(); ScaLBL_Comm->Barrier();
//************************************************************************ //************************************************************************
analysis.basic(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); // allow initial ramp-up to get closer to steady state
CURRENT_TIMESTEP += 2;
if (CURRENT_TIMESTEP > MIN_STEADY_TIMESTEPS){
analysis.finish();
double volB = Averages->gwb.V;
double volA = Averages->gnb.V;
volA /= Dm->Volume;
volB /= Dm->Volume;;
//initial_volume = volA*Dm->Volume;
double vA_x = Averages->gnb.Px/Averages->gnb.M;
double vA_y = Averages->gnb.Py/Averages->gnb.M;
double vA_z = Averages->gnb.Pz/Averages->gnb.M;
double vB_x = Averages->gwb.Px/Averages->gwb.M;
double vB_y = Averages->gwb.Py/Averages->gwb.M;
double vB_z = Averages->gwb.Pz/Averages->gwb.M;
double muA = rhoA*(tauA-0.5)/3.f;
double muB = rhoB*(tauB-0.5)/3.f;
double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz);
double dir_x = Fx/force_mag;
double dir_y = Fy/force_mag;
double dir_z = Fz/force_mag;
if (force_mag == 0.0){
// default to z direction
dir_x = 0.0;
dir_y = 0.0;
dir_z = 1.0;
force_mag = 1.0;
}
double current_saturation = volB/(volA+volB);
double flow_rate_A = volA*(vA_x*dir_x + vA_y*dir_y + vA_z*dir_z);
double flow_rate_B = volB*(vB_x*dir_x + vB_y*dir_y + vB_z*dir_z);
double Ca = fabs(muA*flow_rate_A + muB*flow_rate_B)/(5.796*alpha);
bool isSteady = false;
if ( (fabs((Ca - Ca_previous)/Ca) < tolerance && CURRENT_TIMESTEP > MIN_STEADY_TIMESTEPS))
isSteady = true;
if (CURRENT_TIMESTEP >= MAX_STEADY_TIMESTEPS)
isSteady = true;
if (RESCALE_FORCE == true && SET_CAPILLARY_NUMBER == true && CURRENT_TIMESTEP > RESCALE_FORCE_AFTER_TIMESTEP){
RESCALE_FORCE = false;
double RESCALE_FORCE_FACTOR = capillary_number / Ca;
if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0;
if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5;
Fx *= RESCALE_FORCE_FACTOR;
Fy *= RESCALE_FORCE_FACTOR;
Fz *= RESCALE_FORCE_FACTOR;
force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz);
if (force_mag > 1e-3){
Fx *= 1e-3/force_mag; // impose ceiling for stability
Fy *= 1e-3/force_mag;
Fz *= 1e-3/force_mag;
}
if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca);
Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta);
color_db->putVector<double>("F",{Fx,Fy,Fz});
}
if ( isSteady ){
Averages->Full();
Averages->Write(timestep);
analysis.WriteVisData(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den );
analysis.finish();
if (rank==0){
printf("** WRITE STEADY POINT *** ");
printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous);
double h = Dm->voxel_length;
// pressures
double pA = Averages->gnb.p;
double pB = Averages->gwb.p;
double pAc = Averages->gnc.p;
double pBc = Averages->gwc.p;
double pAB = (pA-pB)/(h*6.0*alpha);
double pAB_connected = (pAc-pBc)/(h*6.0*alpha);
// connected contribution
double Vol_nc = Averages->gnc.V/Dm->Volume;
double Vol_wc = Averages->gwc.V/Dm->Volume;
double Vol_nd = Averages->gnd.V/Dm->Volume;
double Vol_wd = Averages->gwd.V/Dm->Volume;
double Mass_n = Averages->gnc.M + Averages->gnd.M;
double Mass_w = Averages->gwc.M + Averages->gwd.M;
double vAc_x = Averages->gnc.Px/Mass_n;
double vAc_y = Averages->gnc.Py/Mass_n;
double vAc_z = Averages->gnc.Pz/Mass_n;
double vBc_x = Averages->gwc.Px/Mass_w;
double vBc_y = Averages->gwc.Py/Mass_w;
double vBc_z = Averages->gwc.Pz/Mass_w;
// disconnected contribution
double vAd_x = Averages->gnd.Px/Mass_n;
double vAd_y = Averages->gnd.Py/Mass_n;
double vAd_z = Averages->gnd.Pz/Mass_n;
double vBd_x = Averages->gwd.Px/Mass_w;
double vBd_y = Averages->gwd.Py/Mass_w;
double vBd_z = Averages->gwd.Pz/Mass_w;
double flow_rate_A_connected = Vol_nc*(vAc_x*dir_x + vAc_y*dir_y + vAc_z*dir_z);
double flow_rate_B_connected = Vol_wc*(vBc_x*dir_x + vBc_y*dir_y + vBc_z*dir_z);
double flow_rate_A_disconnected = (Vol_nd)*(vAd_x*dir_x + vAd_y*dir_y + vAd_z*dir_z);
double flow_rate_B_disconnected = (Vol_wd)*(vBd_x*dir_x + vBd_y*dir_y + vBd_z*dir_z);
double kAeff_connected = h*h*muA*flow_rate_A_connected/(force_mag);
double kBeff_connected = h*h*muB*flow_rate_B_connected/(force_mag);
double kAeff_disconnected = h*h*muA*flow_rate_A_disconnected/(force_mag);
double kBeff_disconnected = h*h*muB*flow_rate_B_disconnected/(force_mag);
double kAeff = h*h*muA*(flow_rate_A)/(force_mag);
double kBeff = h*h*muB*(flow_rate_B)/(force_mag);
double viscous_pressure_drop = (rhoA*volA + rhoB*volB)*force_mag;
double Mobility = muA/muB;
bool WriteHeader=false;
FILE * kr_log_file = fopen("relperm.csv","r");
if (kr_log_file != NULL)
fclose(kr_log_file);
else
WriteHeader=true;
kr_log_file = fopen("relperm.csv","a");
if (WriteHeader)
fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water eff.perm.oil.connected eff.perm.water.connected eff.perm.oil.disconnected eff.perm.water.disconnected cap.pressure cap.pressure.connected pressure.drop Ca M\n");
fprintf(kr_log_file,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",CURRENT_TIMESTEP,current_saturation,kAeff,kBeff,kAeff_connected,kBeff_connected,kAeff_disconnected,kBeff_disconnected,pAB,pAB_connected,viscous_pressure_drop,Ca,Mobility);
fclose(kr_log_file);
printf(" Measured capillary number %f \n ",Ca);
}
if (SET_CAPILLARY_NUMBER ){
Fx *= capillary_number / Ca;
Fy *= capillary_number / Ca;
Fz *= capillary_number / Ca;
if (force_mag > 1e-3){
Fx *= 1e-3/force_mag; // impose ceiling for stability
Fy *= 1e-3/force_mag;
Fz *= 1e-3/force_mag;
}
if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca);
Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta);
color_db->putVector<double>("F",{Fx,Fy,Fz});
}
else{
if (rank==0){
printf("** Continue to simulate steady *** \n ");
printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous);
}
}
}
}
} }
analysis.finish();
PROFILE_STOP("Update"); PROFILE_STOP("Update");
PROFILE_STOP("Loop"); PROFILE_STOP("Loop");
@ -802,6 +985,13 @@ void ScaLBL_ColorModel::Run(){
printf(" tolerance = %f \n",tolerance); printf(" tolerance = %f \n",tolerance);
printf(" morph_delta = %f \n",morph_delta); printf(" morph_delta = %f \n",morph_delta);
} }
else if (protocol == "fractional flow"){
printf(" using protocol = fractional flow \n");
printf(" min_steady_timesteps = %i \n",MIN_STEADY_TIMESTEPS);
printf(" max_steady_timesteps = %i \n",MAX_STEADY_TIMESTEPS);
printf(" tolerance = %f \n",tolerance);
printf(" morph_delta = %f \n",morph_delta);
}
printf("No. of timesteps: %i \n", timestepMax); printf("No. of timesteps: %i \n", timestepMax);
fflush(stdout); fflush(stdout);
} }
@ -1736,6 +1926,166 @@ FlowAdaptor::~FlowAdaptor(){
} }
double FlowAdaptor::UpdateFractionalFlow(ScaLBL_ColorModel &M){
double MASS_FRACTION_CHANGE = 0.05;
if (M.db->keyExists( "FlowAdaptor" )){
auto flow_db = M.db->getDatabase( "FlowAdaptor" );
MASS_FRACTION_CHANGE = flow_db->getWithDefault<double>( "fractional_flow_increment", 0.05);
}
int Np = M.Np;
double dA, dB, phi;
double vx,vy,vz;
double vax,vay,vaz;
double vbx,vby,vbz;
double vax_global,vay_global,vaz_global;
double vbx_global,vby_global,vbz_global;
double mass_a, mass_b, mass_a_global, mass_b_global;
double *Aq_tmp, *Bq_tmp;
double *Vel_x, *Vel_y, *Vel_z, *Phase;
Aq_tmp = new double [7*Np];
Bq_tmp = new double [7*Np];
Phase = new double [Np];
Vel_x = new double [Np];
Vel_y = new double [Np];
Vel_z = new double [Np];
ScaLBL_CopyToHost(Aq_tmp, M.Aq, 7*Np*sizeof(double));
ScaLBL_CopyToHost(Bq_tmp, M.Bq, 7*Np*sizeof(double));
ScaLBL_CopyToHost(Vel_x, &M.Velocity[0], Np*sizeof(double));
ScaLBL_CopyToHost(Vel_y, &M.Velocity[Np], Np*sizeof(double));
ScaLBL_CopyToHost(Vel_z, &M.Velocity[2*Np], Np*sizeof(double));
/* compute the total momentum */
vax = vay = vaz = 0.0;
vbx = vby = vbz = 0.0;
mass_a = mass_b = 0.0;
for (int n=0; n < M.ScaLBL_Comm->LastExterior(); n++){
dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np];
dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np];
phi = (dA - dB) / (dA + dB);
Phase[n] = phi;
mass_a += dA;
mass_b += dB;
if (phi > 0.0){
vax += Vel_x[n];
vay += Vel_y[n];
vaz += Vel_z[n];
}
else {
vbx += Vel_x[n];
vby += Vel_y[n];
vbz += Vel_z[n];
}
}
for (int n=M.ScaLBL_Comm->FirstInterior(); n < M.ScaLBL_Comm->LastInterior(); n++){
dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np];
dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np];
phi = (dA - dB) / (dA + dB);
Phase[n] = phi;
mass_a += dA;
mass_b += dB;
if (phi > 0.0){
vax += Vel_x[n];
vay += Vel_y[n];
vaz += Vel_z[n];
}
else {
vbx += Vel_x[n];
vby += Vel_y[n];
vbz += Vel_z[n];
}
}
mass_a_global = M.Dm->Comm.sumReduce(mass_a);
mass_b_global = M.Dm->Comm.sumReduce(mass_b);
vax_global = M.Dm->Comm.sumReduce(vax);
vay_global = M.Dm->Comm.sumReduce(vay);
vaz_global = M.Dm->Comm.sumReduce(vaz);
vbx_global = M.Dm->Comm.sumReduce(vbx);
vby_global = M.Dm->Comm.sumReduce(vby);
vbz_global = M.Dm->Comm.sumReduce(vbz);
double total_momentum_A = sqrt(vax_global*vax_global+vay_global*vay_global+vaz_global*vaz_global);
double total_momentum_B = sqrt(vbx_global*vbx_global+vby_global*vby_global+vbz_global*vbz_global);
/* compute the total mass change */
double TOTAL_MASS_CHANGE = MASS_FRACTION_CHANGE*(mass_a_global + mass_b_global);
if (fabs(TOTAL_MASS_CHANGE) > 0.1*mass_a_global )
TOTAL_MASS_CHANGE = 0.1*mass_a_global;
if (fabs(TOTAL_MASS_CHANGE) > 0.1*mass_b_global )
TOTAL_MASS_CHANGE = 0.1*mass_b_global;
double LOCAL_MASS_CHANGE = 0.0;
for (int n=0; n < M.ScaLBL_Comm->LastExterior(); n++){
phi = Phase[n];
vx = Vel_x[n];
vy = Vel_y[n];
vz = Vel_z[n];
double local_momentum = sqrt(vx*vx+vy*vy+vz*vz);
if (phi > 0.0){
LOCAL_MASS_CHANGE = TOTAL_MASS_CHANGE*local_momentum/total_momentum_A;
Aq_tmp[n] -= 0.3333333333333333*LOCAL_MASS_CHANGE;
Aq_tmp[n+Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
Aq_tmp[n+2*Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
Aq_tmp[n+3*Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
Aq_tmp[n+4*Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
Aq_tmp[n+5*Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
Aq_tmp[n+6*Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
}
else{
LOCAL_MASS_CHANGE = TOTAL_MASS_CHANGE*local_momentum/total_momentum_B;
Bq_tmp[n] += 0.3333333333333333*LOCAL_MASS_CHANGE;
Bq_tmp[n+Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
Bq_tmp[n+2*Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
Bq_tmp[n+3*Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
Bq_tmp[n+4*Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
Bq_tmp[n+5*Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
Bq_tmp[n+6*Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
}
}
for (int n=M.ScaLBL_Comm->FirstInterior(); n < M.ScaLBL_Comm->LastInterior(); n++){
phi = Phase[n];
vx = Vel_x[n];
vy = Vel_y[n];
vz = Vel_z[n];
double local_momentum = sqrt(vx*vx+vy*vy+vz*vz);
if (phi > 0.0){
LOCAL_MASS_CHANGE = TOTAL_MASS_CHANGE*local_momentum/total_momentum_A;
Aq_tmp[n] -= 0.3333333333333333*LOCAL_MASS_CHANGE;
Aq_tmp[n+Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
Aq_tmp[n+2*Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
Aq_tmp[n+3*Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
Aq_tmp[n+4*Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
Aq_tmp[n+5*Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
Aq_tmp[n+6*Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
}
else{
LOCAL_MASS_CHANGE = TOTAL_MASS_CHANGE*local_momentum/total_momentum_B;
Bq_tmp[n] += 0.3333333333333333*LOCAL_MASS_CHANGE;
Bq_tmp[n+Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
Bq_tmp[n+2*Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
Bq_tmp[n+3*Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
Bq_tmp[n+4*Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
Bq_tmp[n+5*Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
Bq_tmp[n+6*Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
}
}
if (M.rank == 0) printf("Update Fractional Flow: change mass of fluid B by %f \n",TOTAL_MASS_CHANGE/mass_b_global);
// Need to initialize Aq, Bq, Den, Phi directly
//ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double));
ScaLBL_CopyToDevice(M.Aq, Aq_tmp, 7*Np*sizeof(double));
ScaLBL_CopyToDevice(M.Bq, Bq_tmp, 7*Np*sizeof(double));
return(TOTAL_MASS_CHANGE);
}
double FlowAdaptor::MoveInterface(ScaLBL_ColorModel &M){ double FlowAdaptor::MoveInterface(ScaLBL_ColorModel &M){
double INTERFACE_CUTOFF = M.color_db->getWithDefault<double>( "move_interface_cutoff", 0.975 ); double INTERFACE_CUTOFF = M.color_db->getWithDefault<double>( "move_interface_cutoff", 0.975 );

View File

@ -97,6 +97,7 @@ public:
FlowAdaptor(ScaLBL_ColorModel &M); FlowAdaptor(ScaLBL_ColorModel &M);
~FlowAdaptor(); ~FlowAdaptor();
double MoveInterface(ScaLBL_ColorModel &M); double MoveInterface(ScaLBL_ColorModel &M);
double UpdateFractionalFlow(ScaLBL_ColorModel &M);
DoubleArray phi; DoubleArray phi;
DoubleArray phi_t; DoubleArray phi_t;
private: private:

View File

@ -5,19 +5,35 @@
#include "analysis/distance.h" #include "analysis/distance.h"
#include "common/ReadMicroCT.h" #include "common/ReadMicroCT.h"
ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, const Utilities::MPI& COMM):
rank(RANK), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),tolerance(0),h(0), static inline bool fileExists( const std::string &filename )
epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Vin(0),Vout(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0),
chargeDen_dummy(0),WriteLog(0),nprocx(0),nprocy(0),nprocz(0),
BoundaryConditionInlet(0),BoundaryConditionOutlet(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),
Vin0(0),freqIn(0),t0_In(0),Vin_Type(0),Vout0(0),freqOut(0),t0_Out(0),Vout_Type(0),
TestPeriodic(0),TestPeriodicTime(0),TestPeriodicTimeConv(0),TestPeriodicSaveInterval(0),
comm(COMM)
{ {
std::ifstream ifile( filename.c_str() );
return ifile.good();
} }
ScaLBL_Poisson::~ScaLBL_Poisson(){
ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, const Utilities::MPI& COMM):
rank(RANK), TIMELOG(nullptr), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),tolerance(0),h(0),
epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Vin(0),Vout(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0),
chargeDen_dummy(0),WriteLog(0),nprocx(0),nprocy(0),nprocz(0),
BoundaryConditionInlet(0),BoundaryConditionOutlet(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),
Vin0(0),freqIn(0),t0_In(0),Vin_Type(0),Vout0(0),freqOut(0),t0_Out(0),Vout_Type(0),
TestPeriodic(0),TestPeriodicTime(0),TestPeriodicTimeConv(0),TestPeriodicSaveInterval(0),
comm(COMM)
{
if ( rank == 0 ) {
bool WriteHeader = !fileExists( "PoissonSolver_Convergence.csv" );
TIMELOG = fopen("PoissonSolver_Convergence.csv","a+");
if (WriteHeader)
fprintf(TIMELOG,"Timestep Error\n");
}
}
ScaLBL_Poisson::~ScaLBL_Poisson()
{
if ( TIMELOG )
fclose( TIMELOG );
} }
void ScaLBL_Poisson::ReadParams(string filename){ void ScaLBL_Poisson::ReadParams(string filename){
@ -217,20 +233,19 @@ void ScaLBL_Poisson::ReadInput(){
void ScaLBL_Poisson::AssignSolidBoundary(double *poisson_solid) void ScaLBL_Poisson::AssignSolidBoundary(double *poisson_solid)
{ {
size_t NLABELS=0;
signed char VALUE=0; signed char VALUE=0;
double AFFINITY=0.f; double AFFINITY=0.f;
auto LabelList = electric_db->getVector<int>( "SolidLabels" ); auto LabelList = electric_db->getVector<int>( "SolidLabels" );
auto AffinityList = electric_db->getVector<double>( "SolidValues" ); auto AffinityList = electric_db->getVector<double>( "SolidValues" );
NLABELS=LabelList.size(); size_t NLABELS = LabelList.size();
if (NLABELS != AffinityList.size()){ if (NLABELS != AffinityList.size()){
ERROR("Error: LB-Poisson Solver: SolidLabels and SolidValues must be the same length! \n"); ERROR("Error: LB-Poisson Solver: SolidLabels and SolidValues must be the same length! \n");
} }
double label_count[NLABELS]; std::vector<double> label_count( NLABELS, 0.0 );
double label_count_global[NLABELS]; std::vector<double> label_count_global( NLABELS, 0.0 );
// Assign the labels // Assign the labels
for (size_t idx=0; idx<NLABELS; idx++) label_count[idx]=0; for (size_t idx=0; idx<NLABELS; idx++) label_count[idx]=0;
@ -595,26 +610,11 @@ void ScaLBL_Poisson::Run(double *ChargeDensity, int timestep_from_Study){
} }
void ScaLBL_Poisson::getConvergenceLog(int timestep,double error){
if (rank==0){
bool WriteHeader=false;
TIMELOG = fopen("PoissonSolver_Convergence.csv","r");
if (TIMELOG != NULL)
fclose(TIMELOG);
else
WriteHeader=true;
TIMELOG = fopen("PoissonSolver_Convergence.csv","a+"); void ScaLBL_Poisson::getConvergenceLog(int timestep,double error){
if (WriteHeader) if ( rank == 0 ) {
{ fprintf(TIMELOG,"%i %.5g\n",timestep,error);
fprintf(TIMELOG,"Timestep Error\n"); fflush(TIMELOG);
fprintf(TIMELOG,"%i %.5g\n",timestep,error);
fflush(TIMELOG);
}
else {
fprintf(TIMELOG,"%i %.5g\n",timestep,error);
fflush(TIMELOG);
}
} }
} }

View File

@ -89,12 +89,13 @@ public:
private: private:
Utilities::MPI comm; Utilities::MPI comm;
FILE *TIMELOG;
// filenames // filenames
char LocalRankString[8]; char LocalRankString[8];
char LocalRankFilename[40]; char LocalRankFilename[40];
char LocalRestartFile[40]; char LocalRestartFile[40];
char OutputFilename[200]; char OutputFilename[200];
FILE *TIMELOG;
//int rank,nprocs; //int rank,nprocs;
void LoadParams(std::shared_ptr<Database> db0); void LoadParams(std::shared_ptr<Database> db0);

View File

@ -0,0 +1,33 @@
export TPL_ROOT=/ccs/home/mbt/repos
export TPL_BUILDER=/ccs/home/mbt/repos/TPL-builder
export TPL_WEBPAGE=http://bitbucket.org/AdvancedMultiPhysics/tpl-builder/downloads
export INSTALL_DIR=/ccs/home/mbt/spock/install
module load cmake
module load llvm-amdgpu
module load hip
cmake \
-D CMAKE_BUILD_TYPE=Release \
-D CXX_STD=14 \
-D TPL_INSTALL_DIR:PATH="${INSTALL_DIR}" \
-D C_COMPILER=cc \
-D CXX_COMPILER=CC \
-D Fortran_COMPILER=ftn \
-D FFLAGS="-fPIC" \
-D CFLAGS="-fPIC" \
-D CXXFLAGS="-fPIC" \
-D LDFLAGS="" \
-D ENABLE_STATIC:BOOL=ON \
-D ENABLE_SHARED:BOOL=OFF \
-D PROCS_INSTALL=8 \
-D TPL_LIST:STRING="TIMER;ZLIB;HDF5;SILO" \
-D TIMER_URL="${TPL_ROOT}/TimerUtility" \
-D ZLIB_URL="http://zlib.net/zlib-1.2.11.tar.gz" \
-D HDF5_URL="${TPL_ROOT}/hdf5-1.8.12.tar.gz" \
-D BUILD_TYPE=x86_64 \
-D SILO_URL="${TPL_ROOT}/silo-4.10.2.tar.gz" \
${TPL_BUILDER}

View File

@ -0,0 +1,30 @@
module load cmake
module load llvm-amdgpu
module load hip
export TPL_DIR=/gpfs/alpine/stf006/proj-shared/mbt/spock/install
# configure
rm -rf CMake*
cmake \
-D CMAKE_BUILD_TYPE:STRING=Release \
-D CMAKE_C_COMPILER:PATH=cc \
-D CMAKE_CXX_COMPILER:PATH=CC \
-D CMAKE_CXX_STANDARD=14 \
-D USE_HIP=1 \
-D LINK_LIBRARIES=${HIP_PATH}/lib/libamdhip64.so \
-D USE_CUDA=0 \
-D CMAKE_CUDA_FLAGS="-arch sm_70 -Xptxas=-v -Xptxas -dlcm=cg -lineinfo" \
-D CMAKE_CUDA_HOST_COMPILER="gcc" \
-D USE_MPI=0 \
-D USE_HDF5=1 \
-D HDF5_DIRECTORY="${TPL_DIR}/hdf5" \
-D USE_SILO=0 \
-D SILO_DIRECTORY="${TPL_DIR}/silo" \
-D USE_DOXYGEN:BOOL=false \
-D USE_TIMER=0 \
~/repos/LBPM-WIA

View File

@ -71,21 +71,38 @@ int main( int argc, char **argv )
if (SimulationMode == "development"){ if (SimulationMode == "development"){
double MLUPS=0.0; double MLUPS=0.0;
int timestep = 0; int timestep = 0;
int analysis_interval = ColorModel.timestepMax; /* flow adaptor keys to control */
if (ColorModel.analysis_db->keyExists( "" )){ auto flow_db = ColorModel.db->getDatabase( "FlowAdaptor" );
analysis_interval = ColorModel.analysis_db->getScalar<int>( "analysis_interval" ); int MAX_STEADY_TIME = flow_db->getWithDefault<int>( "max_steady_timesteps", 1000000 );
int SKIP_TIMESTEPS = flow_db->getWithDefault<int>( "skip_timesteps", 100000 );
int ANALYSIS_INTERVAL = ColorModel.timestepMax;
if (ColorModel.analysis_db->keyExists( "analysis_interval" )){
ANALYSIS_INTERVAL = ColorModel.analysis_db->getScalar<int>( "analysis_interval" );
} }
/* Launch the simulation */
FlowAdaptor Adapt(ColorModel); FlowAdaptor Adapt(ColorModel);
runAnalysis analysis(ColorModel); runAnalysis analysis(ColorModel);
while (ColorModel.timestep < ColorModel.timestepMax){ while (ColorModel.timestep < ColorModel.timestepMax){
timestep += analysis_interval; /* this will run steady points */
timestep += MAX_STEADY_TIME;
MLUPS = ColorModel.Run(timestep); MLUPS = ColorModel.Run(timestep);
if (rank==0) printf("Lattice update rate (per MPI process)= %f MLUPS \n", MLUPS); if (rank==0) printf("Lattice update rate (per MPI process)= %f MLUPS \n", MLUPS);
Adapt.UpdateFractionalFlow(ColorModel);
Adapt.MoveInterface(ColorModel); /* apply timestep skipping algorithm to accelerate steady-state */
int skip_time = 0;
timestep = ColorModel.timestep;
while (skip_time < SKIP_TIMESTEPS){
timestep += ANALYSIS_INTERVAL;
MLUPS = ColorModel.Run(timestep);
Adapt.MoveInterface(ColorModel);
skip_time += ANALYSIS_INTERVAL;
}
} }
ColorModel.WriteDebug(); ColorModel.WriteDebug();
} //Analysis.WriteVis(LeeModel,LeeModel.db, timestep); }
else else
ColorModel.Run(); ColorModel.Run();