merge FOM into FOM_dev

CMakeLists.txt

@@ -1,5 +1,8 @@
 # Set some CMake properties    
 CMAKE_MINIMUM_REQUIRED( VERSION 3.9 )
+if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.20.0") 
+    CMAKE_POLICY( SET CMP0115 OLD )
+endif()
 
 
 MESSAGE("====================")

IO/IOHelpers.h

@@ -57,6 +57,6 @@ inline std::vector<std::string> splitList( const char *line, const char token )
 }
 
 
-}; // namespace IO
+} // namespace IO
 
 #endif

IO/silo.h

@@ -18,7 +18,8 @@ typedef int DBfile;
 #endif
 
 
-namespace IO::silo {
+namespace IO {
+namespace silo {
 
 
 enum FileMode { READ, WRITE, CREATE };
@@ -256,7 +257,9 @@ void writeMultiVar( DBfile *fid, const std::string &varname,
     const std::vector<std::string> &subVarNames, const std::vector<int> &subVarTypes );
 
 
-}; // namespace IO::silo
+} // namespace silo
+} // namespace IO
+
 #endif
 
 #include "IO/silo.hpp"

IO/silo.hpp

@@ -13,7 +13,8 @@
 #include <silo.h>
 
 
-namespace IO::silo {
+namespace IO {
+namespace silo {
 
 
 /****************************************************
@@ -413,7 +414,8 @@ Array<TYPE> readTriMeshVariable( DBfile *fid, const std::string &varname )
 }
 
 
-}; // namespace IO::silo
+} // namespace silo
+} // namespace IO
 
 
 #endif

cmake/macros.cmake

@@ -1,3 +1,9 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 3.18.3)
+CMAKE_POLICY( SET CMP0057 NEW )
+if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.20.0") 
+    CMAKE_POLICY( SET CMP0115 OLD )
+endif()
+
 INCLUDE(CheckCCompilerFlag)
 INCLUDE(CheckCSourceCompiles)
 INCLUDE(CheckCXXCompilerFlag)
@@ -47,6 +53,18 @@ ELSEIF( NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION )
 ENDIF()
 
 
+# Check for gold linker
+IF ( UNIX AND NOT APPLE AND NOT DISABLE_GOLD )
+    EXECUTE_PROCESS( COMMAND ${CMAKE_CXX_COMPILER} -fuse-ld=gold -Wl,--version ERROR_QUIET OUTPUT_VARIABLE LD_VERSION )
+    IF ( "${LD_VERSION}" MATCHES "GNU gold" )
+        MESSAGE( "Using gold linker" )
+        SET( GOLD_LINK_FLAGS "-fuse-ld=gold -Wl,--disable-new-dtags -Wl,--no-map-whole-files" )
+        SET( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GOLD_LINK_FLAGS}" )
+        SET( CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${GOLD_LINK_FLAGS}" )
+    ENDIF()
+ENDIF()
+
+
 # Add some default targets if they do not exist
 IF ( NOT TARGET copy-${PROJ}-Data )
     ADD_CUSTOM_TARGET( copy-${PROJ}-Data ALL )
@@ -75,7 +93,7 @@ ENDMACRO()
 MACRO( PRINT_ALL_VARIABLES )
     GET_CMAKE_PROPERTY(_variableNames VARIABLES)
     FOREACH(_variableName ${_variableNames})
-        message(STATUS "${_variableName}=${${_variableName}}")
+        MESSAGE(STATUS "${_variableName}=${${_variableName}}")
     ENDFOREACH()
 ENDMACRO()
 
@@ -115,21 +133,21 @@ MACRO( CONVERT_M4_FORTRAN IN LOCAL_PATH OUT_PATH )
         STRING(REGEX REPLACE "\\\\" "" COMPILE_CMD "${COMPILE_CMD}")
         MESSAGE("COMPILE_CMD = ${COMPILE_CMD}")
         SET( COMPILE_CMD ${COMPILE_CMD} )
-        add_custom_command(
+        ADD_CUSTOM_COMMAND(
             OUTPUT ${OUT2}
             COMMAND m4 -I${LOCAL_PATH} -I${SAMRAI_FORTDIR} ${M4DIRS} ${IN} > ${OUT}
             COMMAND ${COMPILE_CMD}
             DEPENDS ${IN}
             )
-        set_source_files_properties(${OUT2} PROPERTIES GENERATED true)
+        SET_SOURCE_FILES_PROPERTIES(${OUT2} PROPERTIES GENERATED true)
         SET( SOURCES ${SOURCES} "${OUT2}" )
      ELSE()
-        add_custom_command(
+        ADD_CUSTOM_COMMAND(
             OUTPUT ${OUT}
             COMMAND m4 -I${LOCAL_PATH} -I${SAMRAI_FORTDIR} ${M4DIRS} ${M4_OPTIONS} ${IN} > ${OUT}
             DEPENDS ${IN}
             )
-         set_source_files_properties(${OUT} PROPERTIES GENERATED true)
+         SET_SOURCE_FILES_PROPERTIES(${OUT} PROPERTIES GENERATED true)
          SET( SOURCES ${SOURCES} "${OUT}" )
      ENDIF()
 ENDMACRO()
@@ -145,6 +163,10 @@ ENDMACRO()
 
 # Add a project executable
 MACRO( ADD_${PROJ}_EXECUTABLE EXEFILE )
+    IF ( TARGET ${EXEFILE} AND NOT INCLUDE_${EXEFILE} )
+        MESSAGE( FATAL_ERROR "ADD_${PROJ}_EXECUTABLE should be called before adding tests" )
+    ENDIF()
+    SET( INCLUDE_${EXEFILE} TRUE )
     ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
     INSTALL( TARGETS ${EXEFILE} DESTINATION ${${PROJ}_INSTALL_DIR}/bin )
 ENDMACRO()
@@ -193,7 +215,7 @@ MACRO (FIND_FILES)
     SET( CSOURCES ${CSOURCES} ${T_CSOURCES} )
     SET( FSOURCES ${FSOURCES} ${T_FSOURCES} )
     SET( M4FSOURCES ${M4FSOURCES} ${T_M4FSOURCES} )
-    SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${T_M4FSOURCES} )
+    SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} ${T_FSOURCES} ${T_M4FSOURCES} ${CUDASOURCES} )
 ENDMACRO()
 
 
@@ -420,6 +442,9 @@ MACRO( IDENTIFY_COMPILER )
         ELSEIF ( ${CMAKE_Fortran_COMPILER_ID} MATCHES "PGI")
             SET(USING_PGF90 TRUE)
             MESSAGE("Using pgf90")
+        ELSEIF( (${CMAKE_Fortran_COMPILER_ID} MATCHES "CRAY") OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "Cray") )
+            SET(USING_CRAY TRUE)
+            MESSAGE("Using Cray")
         ELSEIF ( (${CMAKE_Fortran_COMPILER_ID} MATCHES "CLANG") OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "Clang") OR
                  (${CMAKE_Fortran_COMPILER_ID} MATCHES "FLANG") OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "Flang") )
             SET(USING_FLANG TRUE)
@@ -440,7 +465,7 @@ MACRO( SET_WARNINGS )
     # Note: adding -Wlogical-op causes a wierd linking error on Titan using the nvcc wrapper:
     #    /usr/bin/ld: cannot find gical-op: No such file or directory
     SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -Wall -Wextra") 
-    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Woverloaded-virtual -Wsign-compare")
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -Woverloaded-virtual -Wsign-compare -Wformat-security")
   ELSEIF ( USING_MSVC )
     # Add Microsoft specifc compiler options
     SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} /D _SCL_SECURE_NO_WARNINGS /D _CRT_SECURE_NO_WARNINGS /D _ITERATOR_DEBUG_LEVEL=0 /wd4267" )
@@ -448,7 +473,7 @@ MACRO( SET_WARNINGS )
   ELSEIF ( USING_ICC )
     # Add Intel specifc compiler options
     SET(CMAKE_C_FLAGS     " ${CMAKE_C_FLAGS} -Wall" )
-    SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall" )
+    SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall -wd1011" )
   ELSEIF ( USING_CRAY )
     # Add default compiler options
     SET(CMAKE_C_FLAGS     " ${CMAKE_C_FLAGS}")
@@ -461,8 +486,8 @@ MACRO( SET_WARNINGS )
     SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} --diag_suppress 111,128,185")
   ELSEIF ( USING_CLANG )
     # Add default compiler options
-    SET(CMAKE_C_FLAGS     " ${CMAKE_C_FLAGS} -Wall")
+    SET(CMAKE_C_FLAGS     " ${CMAKE_C_FLAGS} -Wall -Wextra")
-    SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall -Wno-missing-braces -Wmissing-field-initializers -ftemplate-depth=1024")
+    SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic -Wno-missing-braces -Wmissing-field-initializers -ftemplate-depth=1024")
   ELSEIF ( USING_XL )
     # Add default compiler options
     SET(CMAKE_C_FLAGS     " ${CMAKE_C_FLAGS} -Wall")
@@ -666,25 +691,16 @@ ENDMACRO()
 
 # Link the libraries to the given target
 MACRO( TARGET_LINK_EXTERNAL_LIBRARIES TARGET_NAME )
-    FOREACH ( tmp ${TPL_LIBS} )
+    # Include external libraries
-        TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
-    ENDFOREACH()
     FOREACH ( tmp ${EXTERNAL_LIBS} )
         TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
     ENDFOREACH()
-    FOREACH ( tmp ${LAPACK_LIBS} )
+    # Include libraries found through the TPL builder
+    FOREACH ( tmp ${TPL_LIBS} )
         TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
     ENDFOREACH()
-    FOREACH ( tmp ${BLAS_LIBS} )
+    # Include CMake implicit libraries
-        TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
+    FOREACH ( tmp ${CMAKE_CUDA_IMPLICIT_LINK_LIBRARIES} ${CMAKE_C_IMPLICIT_LINK_LIBRARIES}
-    ENDFOREACH()
-    FOREACH ( tmp ${BLAS_LAPACK_LIBS} )
-        TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
-    ENDFOREACH()
-    FOREACH ( tmp ${MPI_LIBRARIES} )
-        TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
-    ENDFOREACH()
-    FOREACH ( tmp ${CMAKE_C_IMPLICIT_LINK_LIBRARIES}
         ${CMAKE_CXX_IMPLICIT_LINK_LIBRARIES} ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES} )
         TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
     ENDFOREACH()
@@ -783,7 +799,7 @@ FUNCTION( ADD_PROJ_PROVISIONAL_TEST EXEFILE )
         SET( CXXFILE ${EXEFILE} )
         SET( TESTS_SO_FAR ${TESTS_SO_FAR} ${EXEFILE} )
         # Check if we want to add the test to all
-        IF ( NOT EXCLUDE_TESTS_FROM_ALL )
+        IF ( NOT EXCLUDE_TESTS_FROM_ALL OR INCLUDE_${EXEFILE} )
             ADD_EXECUTABLE( ${EXEFILE} ${CXXFILE} )
         ELSE()
             ADD_EXECUTABLE( ${EXEFILE} EXCLUDE_FROM_ALL ${CXXFILE} )
@@ -818,123 +834,218 @@ MACRO( CREATE_TEST_NAME TEST ${ARGN} )
     FOREACH( tmp ${ARGN})
         SET( TESTNAME "${TESTNAME}--${tmp}")
     endforeach()
-    # STRING(REGEX REPLACE "--" "-" TESTNAME ${TESTNAME} )
     SET( LAST_TESTNAME ${TESTNAME} PARENT_SCOPE )
 ENDMACRO()
 
 
 # Function to add the resource locks to an executable
-FUNCTION( ADD_RESOURCE_LOCK TESTNAME EXEFILE ${ARGN} )
+FUNCTION( ADD_RESOURCE TESTNAME RESOURCE )
+    SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES RESOURCE_LOCK ${RESOURCE} )
+ENDFUNCTION()
+FUNCTION( ADD_RESOURCE_LOCKS TESTNAME EXEFILE ${ARGN} )
     IF ( NOT DISABLE_RESOURCE_LOCK )
         IF ( NOT ARGN )
-            SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES RESOURCE_LOCK ${EXEFILE} )
+            ADD_RESOURCE( ${TESTNAME} ${EXEFILE} )
         ELSE()
             FOREACH( tmp ${ARGN} )
-                SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES RESOURCE_LOCK ${tmp} )
+                ADD_RESOURCE( ${TESTNAME} ${tmp} )
             ENDFOREACH()
         ENDIF()
     ENDIF()
 ENDFUNCTION()
 
 
-# Add a executable as a test
+# Function to add a test
-FUNCTION( ADD_${PROJ}_TEST EXEFILE ${ARGN} )
+FUNCTION( CALL_ADD_TEST EXEFILE PROCS THREADS WEEKLY TESTBUILDER ${ARGN} )
     # Check if we actually want to add the test
     KEEP_TEST( RESULT )
     IF ( NOT RESULT )
+        MESSAGE( "Discarding test: ${EXEFILE}" )
         RETURN()
     ENDIF()
+    # Check if we are dealing with a TestBuilder test
+    IF ( ${TESTBUILDER} )
+        # Add the test to the TestBuilder sources
+        SET( TESTBUILDER_SOURCES ${TESTBUILDER_SOURCES} ${EXEFILE} PARENT_SCOPE )
+    ELSE()
         # Add the provisional test
         ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
-    CREATE_TEST_NAME( ${EXEFILE} ${ARGN} )
-    IF ( USE_MPI_FOR_SERIAL_TESTS )
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
-        SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
-    ELSE()
-        ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
     ENDIF()
-    SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS 1 )
+    # Create the test name
-    ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
+    SET( TESTNAME ${EXEFILE} )
-ENDFUNCTION()
+    #IF ( ${TESTBUILDER} )
-
+    #    SET( TESTNAME TB_${TESTNAME} )
-
+    #ENDIF()
-# Add a executable as a weekly test
+    IF ( "${PROCS}" GREATER "1")
-FUNCTION( ADD_${PROJ}_WEEKLY_TEST EXEFILE PROCS ${ARGN} )
+        SET( TESTNAME ${TESTNAME}_${PROCS}procs )
-    # Check if we actually want to add the test
-    KEEP_TEST( RESULT )
-    IF ( NOT RESULT )
-        RETURN()
     ENDIF()
-    # Add the provisional test
+    IF ( "${THREADS}" GREATER "1")
-    ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
+        SET( TESTNAME ${TESTNAME}_${THREADS}threads )
-    IF( ${PROCS} STREQUAL "1" )
-        CREATE_TEST_NAME( "${EXEFILE}_WEEKLY" ${ARGN} )
-    ELSEIF( (USE_MPI OR USE_EXT_MPI) AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) )
-        CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_WEEKLY" ${ARGN} )
     ENDIF()
-    IF ( ${PROCS} GREATER ${TEST_MAX_PROCS} )
+    IF ( ${WEEKLY} )
-        MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})")
+        SET( TESTNAME ${TESTNAME}_WEEKLY )
-    ELSEIF( ${PROCS} STREQUAL "1" )
-        CREATE_TEST_NAME( "${EXEFILE}_WEEKLY" ${ARGN} )
-        IF ( USE_MPI_FOR_SERIAL_TESTS )
-            ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
-            SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
-        ELSE()
-            ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
     ENDIF()
-        SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS 1 )
+    CREATE_TEST_NAME( ${TESTNAME} ${ARGN} )
-        ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
+    # Create the tests for ctest
-    ELSEIF( (USE_MPI OR USE_EXT_MPI) AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) )
-        CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_WEEKLY" ${ARGN} )
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
-        SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )
-        SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
-        ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
-    ENDIF()
-ENDFUNCTION()
-
-
-# Add a executable as a parallel test
-FUNCTION( ADD_${PROJ}_TEST_PARALLEL EXEFILE PROCS ${ARGN} )
-    # Check if we actually want to add the test
-    KEEP_TEST( RESULT )
-    IF ( NOT RESULT )
-        RETURN()
-    ENDIF()
-    # Add the provisional test
-    ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
-    CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs" ${ARGN} )
-    IF ( NOT ( USE_MPI OR USE_EXT_MPI ) )
-        MESSAGE("Disabling test ${TESTNAME} (configured without MPI)")
-    ELSEIF ( ${PROCS} GREATER ${TEST_MAX_PROCS} )
-        MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})")
-    ELSE()
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
-        SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
-        SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )
-        ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
-    ENDIF()
-ENDFUNCTION()
-
-
-# Add a parallel test that may use both MPI and threads
-# This allows us to correctly compute the number of processors used by the test
-MACRO( ADD_${PROJ}_TEST_THREAD_MPI EXEFILE PROCS THREADS ${ARGN} )
-    ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
-    CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_${THREADS}threads" ${ARGN} )
     MATH( EXPR TOT_PROCS "${PROCS} * ${THREADS}" )
-    IF ( ${TOT_PROCS} GREATER ${TEST_MAX_PROCS} )
+    IF ( ${TOT_PROCS} EQUAL 0 )
+        # Skip test (provisional)
+    ELSEIF ( ${TOT_PROCS} GREATER ${TEST_MAX_PROCS} )
         MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})")
     ELSEIF ( ( ${PROCS} STREQUAL "1" ) AND NOT USE_MPI_FOR_SERIAL_TESTS )
+        IF ( ${TESTBUILDER} )
+            ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${TB_TARGET}> ${EXEFILE} ${ARGN} )
+        ELSE()
             ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+        ENDIF()
         SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} )
-        ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
+        ADD_RESOURCE_LOCKS( ${TESTNAME} ${EXEFILE} ${ARGN} )
     ELSEIF ( USE_MPI OR USE_EXT_MPI )
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+        IF ( ${TESTBUILDER} )
+            ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${TB_TARGET}> ${EXEFILE} ${ARGN} )
+        ELSE()
+            ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+        ENDIF()
         SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
         SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} )
-        ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
+        ADD_RESOURCE_LOCKS( ${TESTNAME} ${EXEFILE} ${ARGN} )
     ENDIF()
+ENDFUNCTION()
+
+
+# Functions to create a test builder
+FUNCTION( INITIALIZE_TESTBUILDER )
+    SET( TESTBUILDER_SOURCES PARENT_SCOPE )
+    # Check if we actually want to add the test
+    KEEP_TEST( RESULT )
+    IF ( NOT RESULT )
+        RETURN()
+    ENDIF()
+    # Check if the target does not exist (may not be added yet or we are re-configuring)
+    STRING(REGEX REPLACE "${${PROJ}_SOURCE_DIR}/" "" TB_TARGET "TB-${PROJ}-${CMAKE_CURRENT_SOURCE_DIR}" )
+    STRING(REGEX REPLACE "/" "-" TB_TARGET ${TB_TARGET} )
+    IF ( NOT TARGET ${TB_TARGET} )
+        GLOBAL_SET( ${TB_TARGET}-BINDIR )
+    ENDIF()
+    # Check if test has already been added
+    IF ( NOT ${TB_TARGET}-BINDIR )
+        GLOBAL_SET( ${TB_TARGET}-BINDIR "${CMAKE_CURRENT_BINARY_DIR}" )
+        # Create the initial file
+        IF ( NOT EXISTS  "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" )
+            FILE( WRITE "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" "// Auto generated file\n\n" )
+        ENDIF()
+        # The target has not been added
+        SET( CXXFILE "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" )
+        SET( TESTS_SO_FAR ${TESTS_SO_FAR} ${TB_TARGET} )
+        # Check if we want to add the test to all
+        IF ( NOT EXCLUDE_TESTS_FROM_ALL )
+            ADD_EXECUTABLE( ${TB_TARGET} ${CXXFILE} )
+        ELSE()
+            ADD_EXECUTABLE( ${TB_TARGET} EXCLUDE_FROM_ALL ${CXXFILE} )
+        ENDIF()
+        SET_TARGET_PROPERTIES( ${TB_TARGET} PROPERTIES OUTPUT_NAME "TB" )
+    ELSEIF ( NOT ${TB-BINDIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR} )
+        # We are trying to add 2 different tests with the same name
+        MESSAGE( "Existing test: ${TB-BINDIR}/${TB_TARGET}" )
+        MESSAGE( "New test:      ${CMAKE_CURRENT_BINARY_DIR}/${TB_TARGET}}" )
+        MESSAGE( FATAL_ERROR "Trying to add 2 different tests with the same name" )
+    ENDIF()
+    SET( TB_TARGET ${TB_TARGET} PARENT_SCOPE )
+ENDFUNCTION()
+FUNCTION( FINALIZE_TESTBUILDER )
+    # Check if we actually want to add the test
+    KEEP_TEST( RESULT )
+    IF ( NOT RESULT )
+        RETURN()
+    ENDIF()
+    # Create the library for the test builder
+    IF ( TESTBUILDER_SOURCES )
+        LIST( REMOVE_DUPLICATES TESTBUILDER_SOURCES )
+        SET( TB_TARGET_SOURCES "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" )
+        FOREACH ( tmp ${TESTBUILDER_SOURCES} )
+            GET_FILENAME_COMPONENT( tmp ${tmp} ABSOLUTE )
+            SET( TB_TARGET_SOURCES ${TB_TARGET_SOURCES} ${tmp} )
+        ENDFOREACH()
+        TARGET_SOURCES( ${TB_TARGET} PUBLIC ${TB_TARGET_SOURCES} )
+        ADD_DEPENDENCIES( ${TB_TARGET} copy-${PROJ}-include )
+    ENDIF()
+    # Add the dependencies to the test builder
+    ADD_PROJ_EXE_DEP( ${TB_TARGET} )
+    IF ( CURRENT_LIBRARY )
+        IF ( NOT TARGET ${CURRENT_LIBRARY}-test )
+            ADD_CUSTOM_TARGET( ${CURRENT_LIBRARY}-test )
+        ENDIF()
+        ADD_DEPENDENCIES( ${CURRENT_LIBRARY}-test ${TB_TARGET} )
+    ENDIF()
+    SET( TB_TARGET ${TB_TARGET} PARENT_SCOPE )
+    # Finish generating TestBuilder.cpp
+    SET( TB_FILE "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder-tmp.cpp" )
+    FILE( WRITE "${TB_FILE}" "// Auto generated file\n\n" )
+    FILE( APPEND "${TB_FILE}" "#include <cstring>\n" )
+    FILE( APPEND "${TB_FILE}" "#include <iostream>\n\n" )
+    FOREACH( tmp ${TESTBUILDER_SOURCES} )
+        FILE( APPEND "${TB_FILE}" "extern int ${tmp}( int argc, char *argv[] );\n" )
+    ENDFOREACH()
+    FILE( APPEND "${TB_FILE}" "\n\n// Main Program\n" )
+    FILE( APPEND "${TB_FILE}" "int main( int argc, char *argv[] )\n" )
+    FILE( APPEND "${TB_FILE}" "{\n" )
+    FILE( APPEND "${TB_FILE}" "    if ( argc < 2 ) {\n" )
+    FILE( APPEND "${TB_FILE}" "        std::cerr << \"Invaild number of arguments for TestBuilder\";\n " )
+    FILE( APPEND "${TB_FILE}" "        return 1;\n" )
+    FILE( APPEND "${TB_FILE}" "    }\n\n" )
+    FILE( APPEND "${TB_FILE}" "    int rtn = 0;\n" )
+    FILE( APPEND "${TB_FILE}" "    if ( strcmp( argv[1], \"NULL\" ) == 0 ) {\n" )
+    FOREACH( tmp ${TESTBUILDER_SOURCES} )
+        FILE( APPEND "${TB_FILE}" "    } else if ( strcmp( argv[1], \"${tmp}\" ) == 0 ) {\n" )
+        FILE( APPEND "${TB_FILE}" "        rtn = ${tmp}( argc-1, &argv[1] );\n" )
+    ENDFOREACH()
+    FILE( APPEND "${TB_FILE}" "    } else {\n" )
+    FILE( APPEND "${TB_FILE}" "        std::cerr << \"Unknown test\";\n" )
+    FILE( APPEND "${TB_FILE}" "        return 1;\n" )
+    FILE( APPEND "${TB_FILE}" "    }\n\n" )
+    FILE( APPEND "${TB_FILE}" "    return rtn;\n" )
+    FILE( APPEND "${TB_FILE}" "}\n" )
+    EXECUTE_PROCESS( COMMAND ${CMAKE_COMMAND} -E copy_if_different "${TB_FILE}" "${CMAKE_CURRENT_BINARY_DIR}/TestBuilder.cpp" )
+ENDFUNCTION()
+
+
+# Convience functions to add a test
+MACRO( ADD_${PROJ}_TEST EXEFILE ${ARGN} )
+    CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE FALSE ${ARGN} )
+ENDMACRO()
+MACRO( ADD_${PROJ}_TEST_PARALLEL EXEFILE PROCS ${ARGN} )
+    CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 FALSE FALSE ${ARGN} )
+ENDMACRO()
+MACRO( ADD_${PROJ}_WEEKLY_TEST EXEFILE PROCS ${ARGN} )
+    CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 TRUE FALSE ${ARGN} )
+ENDMACRO()
+MACRO( ADD_${PROJ}_TEST_THREAD_MPI EXEFILE PROCS THREADS ${ARGN} )
+    CALL_ADD_TEST( ${EXEFILE} ${PROCS} ${THREADS} FALSE FALSE ${ARGN} )
+ENDMACRO()
+MACRO( ADD_${PROJ}_TEST_1_2_4 EXEFILE ${ARGN} )
+    CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE FALSE ${ARGN} )
+    CALL_ADD_TEST( ${EXEFILE} 2 1 FALSE FALSE ${ARGN} )
+    CALL_ADD_TEST( ${EXEFILE} 4 1 FALSE FALSE ${ARGN} )
+ENDMACRO()
+MACRO( ADD_TB_PROVISIONAL_TEST EXEFILE ${ARGN} )
+    CALL_ADD_TEST( ${EXEFILE} 0 1 FALSE TRUE ${ARGN} )
+ENDMACRO()
+MACRO( ADD_TB_TEST EXEFILE ${ARGN} )
+    CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE TRUE ${ARGN} )
+ENDMACRO()
+MACRO( ADD_TB_TEST_PARALLEL EXEFILE PROCS ${ARGN} )
+    CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 FALSE TRUE ${ARGN} )
+ENDMACRO()
+MACRO( ADD_TB_WEEKLY_TEST EXEFILE PROCS ${ARGN} )
+    CALL_ADD_TEST( ${EXEFILE} ${PROCS} 1 TRUE TRUE ${ARGN} )
+ENDMACRO()
+MACRO( ADD_TB_TEST_THREAD_MPI EXEFILE PROCS THREADS ${ARGN} )
+    CALL_ADD_TEST( ${EXEFILE} ${PROCS} ${THREADS} FALSE TRUE ${ARGN} )
+ENDMACRO()
+MACRO( ADD_TB_TEST_1_2_4 EXEFILE ${ARGN} )
+    CALL_ADD_TEST( ${EXEFILE} 1 1 FALSE TRUE ${ARGN} )
+    CALL_ADD_TEST( ${EXEFILE} 2 1 FALSE TRUE ${ARGN} )
+    CALL_ADD_TEST( ${EXEFILE} 4 1 FALSE TRUE ${ARGN} )
 ENDMACRO()
 
 
@@ -966,7 +1077,7 @@ FUNCTION( ADD_${PROJ}_EXAMPLE EXEFILE PROCS ${ARGN} )
         ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
     ELSEIF ( USE_EXT_MPI AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) )
         CREATE_TEST_NAME( "example--${EXEFILE}_${PROCS}procs" ${ARGN} )
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
         SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
     ENDIF()
     SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )
@@ -1038,6 +1149,8 @@ FUNCTION( ADD_MATLAB_MEX SOURCE )
     MATLAB_ADD_MEX(
         NAME ${TARGET}
         SRC ${SOURCE}
+        MODULE
+        R2017b
     )
     TARGET_LINK_LIBRARIES( ${TARGET} ${MATLAB_TARGET} )
     ADD_PROJ_EXE_DEP( ${TARGET} )
@@ -1074,7 +1187,9 @@ ENDMACRO()
 
 # Create a script to start matlab preloading libraries
 FUNCTION( CREATE_MATLAB_WRAPPER )
+    SET( MATLABPATH ${MATLABPATH} "${${PROJ}_INSTALL_DIR}/mex" )
     SET( tmp_libs ${MEX_LIBCXX} ${MEX_FILES} )
+    LIST( REMOVE_DUPLICATES MATLABPATH )
     STRING( REGEX REPLACE ";" ":" tmp_libs "${tmp_libs}" )
     STRING( REGEX REPLACE ";" ":" tmp_path "${MATLABPATH}" )
     IF ( USING_MSVC )
@@ -1269,3 +1384,4 @@ FUNCTION( FIND_PYTHON_MODULE MODULE)
     ENDIF()
 	SET( PY_${MODULE2}_FOUND ${PY_${MODULE2}_FOUND} PARENT_SCOPE )
 ENDFUNCTION(FIND_PYTHON_MODULE)
+

common/MPI.I

@@ -595,7 +595,15 @@ template<class TYPE>
 void MPI_CLASS::sendrecv( const TYPE *sendbuf, int sendcount, int dest, int sendtag, 
                           TYPE *recvbuf, int recvcount, int source, int recvtag ) const
 {
-    ERROR( "Not implimented" );
+    if ( getSize() == 1 ) {
+        ASSERT( dest == 0 );
+        ASSERT( source == 0 );
+        ASSERT( sendcount == recvcount );
+        ASSERT( sendtag == recvtag );
+        memcpy( recvbuf, sendbuf, sendcount * sizeof( TYPE ) );
+    } else {
+        ERROR( "Not implimented for " + std::string( typeid( TYPE ).name() ) );
+    }
 }
 #else
 template<class TYPE>

hip/FreeLee.cu

@@ -1,6 +1,7 @@
 #include <math.h>
 #include <stdio.h>
-#include <cuda_profiler_api.h>
+#include "hip/hip_runtime.h"
+#include "hip/hip_runtime_api.h"
 
 #define STOKES
 
@@ -3395,7 +3396,7 @@ extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, doubl
 extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
                                                           double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np)
 {
-	cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField, cudaFuncCachePreferL1);
+	hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField, hipFuncCachePreferL1);
 	dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField<<<NBLOCKS,NTHREADS >>>(neighborList, Map, hq, Den, Phi, ColorGrad, Vel,
              rhoA,  rhoB, tauM, W, start, finish,  Np);
 	hipError_t err = hipGetLastError();
@@ -3407,7 +3408,7 @@ extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map
 extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
 		double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){
 
-	cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField, cudaFuncCachePreferL1);
+	hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField, hipFuncCachePreferL1);
 	dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField<<<NBLOCKS,NTHREADS >>>( Map, hq, Den, Phi, ColorGrad, Vel, rhoA, rhoB, tauM, W, start, finish, Np);
 	hipError_t err = hipGetLastError();
 	if (hipSuccess != err){
@@ -3418,7 +3419,7 @@ extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, dou
 
 extern "C" void ScaLBL_D3Q7_ComputePhaseField(int *Map,  double *hq, double *Den, double *Phi, double rhoA, double rhoB, int start, int finish, int Np){
 
-	cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q7_ComputePhaseField, cudaFuncCachePreferL1);
+	hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q7_ComputePhaseField, hipFuncCachePreferL1);
 	dvc_ScaLBL_D3Q7_ComputePhaseField<<<NBLOCKS,NTHREADS >>>( Map, hq, Den, Phi, rhoA, rhoB, start, finish, Np);
 	hipError_t err = hipGetLastError();
 	if (hipSuccess != err){
@@ -3431,7 +3432,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, dou
                                                 double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, 
                                                 int strideY, int strideZ, int start, int finish, int Np){
 	
-	cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel, cudaFuncCachePreferL1);
+	hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel, hipFuncCachePreferL1);
 	dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel<<<NBLOCKS,NTHREADS >>>(neighborList, Map, dist, Den, Phi, mu_phi, Vel, Pressure,  ColorGrad, 
             rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np);
 	hipError_t err = hipGetLastError();
@@ -3444,7 +3445,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double
                                                 double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, 
                                                 int strideY, int strideZ, int start, int finish, int Np){
 	
-	cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel, cudaFuncCachePreferL1);
+	hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel, hipFuncCachePreferL1);
 	dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel<<<NBLOCKS,NTHREADS >>>(Map, dist, Den, Phi, mu_phi, Vel, Pressure,  ColorGrad, 
             rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np);
 	hipError_t err = hipGetLastError();
@@ -3502,7 +3503,7 @@ extern "C" void ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(int *Map, double *hq,
 extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborList, double *dist, double *Vel, double *Pressure,  
                                                                 double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){
 
-	cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK, cudaFuncCachePreferL1);
+	hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK, hipFuncCachePreferL1);
 	dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK<<<NBLOCKS,NTHREADS >>>(neighborList, dist, Vel, Pressure, 
             tau, rho0, Fx, Fy, Fz, start, finish, Np);
 	hipError_t err = hipGetLastError();
@@ -3514,7 +3515,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborLis
 extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, double *Vel, double *Pressure, 
                                                                  double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){
 
-	cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK, cudaFuncCachePreferL1);
+	hipFuncSetCacheConfig((void*)dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK, hipFuncCachePreferL1);
 	dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK<<<NBLOCKS,NTHREADS >>>(dist, Vel, Pressure, 
             tau, rho0,  Fx, Fy, Fz, start, finish, Np);
 	hipError_t err = hipGetLastError();

models/PoissonSolver.cpp

@@ -5,8 +5,16 @@
 #include "analysis/distance.h"
 #include "common/ReadMicroCT.h"
 
+
+static inline bool fileExists( const std::string &filename )
+{
+    std::ifstream ifile( filename.c_str() );
+    return ifile.good();
+}
+
+
 ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, const Utilities::MPI& COMM):
-rank(RANK), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),tolerance(0),h(0),
+    rank(RANK), TIMELOG(nullptr), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),tolerance(0),h(0),
     epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Vin(0),Vout(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0),
     chargeDen_dummy(0),WriteLog(0),nprocx(0),nprocy(0),nprocz(0),
     BoundaryConditionInlet(0),BoundaryConditionOutlet(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),
@@ -14,10 +22,18 @@ Vin0(0),freqIn(0),t0_In(0),Vin_Type(0),Vout0(0),freqOut(0),t0_Out(0),Vout_Type(0
     TestPeriodic(0),TestPeriodicTime(0),TestPeriodicTimeConv(0),TestPeriodicSaveInterval(0),
     comm(COMM)
 {
+    if ( rank == 0 ) {
+	    bool WriteHeader = !fileExists( "PoissonSolver_Convergence.csv" );
 
+	    TIMELOG = fopen("PoissonSolver_Convergence.csv","a+");
+	    if (WriteHeader)
+		    fprintf(TIMELOG,"Timestep Error\n");
     }
-ScaLBL_Poisson::~ScaLBL_Poisson(){
+}
-
+ScaLBL_Poisson::~ScaLBL_Poisson()
+{
+    if ( TIMELOG )
+        fclose( TIMELOG );
 }
 
 void ScaLBL_Poisson::ReadParams(string filename){
@@ -217,20 +233,19 @@ void ScaLBL_Poisson::ReadInput(){
 
 void ScaLBL_Poisson::AssignSolidBoundary(double *poisson_solid)
 {
-	size_t NLABELS=0;
 	signed char VALUE=0;
 	double AFFINITY=0.f;
 
 	auto LabelList = electric_db->getVector<int>( "SolidLabels" );
 	auto AffinityList = electric_db->getVector<double>( "SolidValues" );
 
-	NLABELS=LabelList.size();
+	size_t NLABELS = LabelList.size();
 	if (NLABELS != AffinityList.size()){
 		ERROR("Error: LB-Poisson Solver: SolidLabels and SolidValues must be the same length! \n");
 	}
 
-	double label_count[NLABELS];
+	std::vector<double> label_count( NLABELS, 0.0 );
-	double label_count_global[NLABELS];
+	std::vector<double> label_count_global( NLABELS, 0.0 );
 	// Assign the labels
 
 	for (size_t idx=0; idx<NLABELS; idx++) label_count[idx]=0;
@@ -595,27 +610,12 @@ void ScaLBL_Poisson::Run(double *ChargeDensity, int timestep_from_Study){
 
 }
 
+
 void ScaLBL_Poisson::getConvergenceLog(int timestep,double error){
     if ( rank == 0 ) {
-		bool WriteHeader=false;
-		TIMELOG = fopen("PoissonSolver_Convergence.csv","r");
-		if (TIMELOG != NULL)
-			fclose(TIMELOG);
-		else
-			WriteHeader=true;
-
-		TIMELOG = fopen("PoissonSolver_Convergence.csv","a+");
-		if (WriteHeader)
-		{
-			fprintf(TIMELOG,"Timestep Error\n");				
         fprintf(TIMELOG,"%i %.5g\n",timestep,error); 
         fflush(TIMELOG);
     }
-        else {
-            fprintf(TIMELOG,"%i %.5g\n",timestep,error); 
-            fflush(TIMELOG);
-        }
-    }
 }
 
 void ScaLBL_Poisson::SolveElectricPotentialAAodd(int timestep_from_Study){

models/PoissonSolver.h

@@ -89,12 +89,13 @@ public:
 private:
 	Utilities::MPI comm;
 	
+	FILE *TIMELOG;
+
 	// filenames
     char LocalRankString[8];
     char LocalRankFilename[40];
     char LocalRestartFile[40];
     char OutputFilename[200];
-	FILE *TIMELOG;
    
     //int rank,nprocs;
     void LoadParams(std::shared_ptr<Database> db0);    	

sample_scripts/configure_spock_TPLs

@@ -0,0 +1,33 @@
+export TPL_ROOT=/ccs/home/mbt/repos
+export TPL_BUILDER=/ccs/home/mbt/repos/TPL-builder
+export TPL_WEBPAGE=http://bitbucket.org/AdvancedMultiPhysics/tpl-builder/downloads
+
+export INSTALL_DIR=/ccs/home/mbt/spock/install
+
+module load cmake
+module load llvm-amdgpu
+module load hip
+
+
+cmake                                                   \
+    -D CMAKE_BUILD_TYPE=Release                         \
+    -D CXX_STD=14                                       \
+    -D TPL_INSTALL_DIR:PATH="${INSTALL_DIR}"            \
+    -D C_COMPILER=cc                                    \
+    -D CXX_COMPILER=CC                                  \
+    -D Fortran_COMPILER=ftn                             \
+    -D FFLAGS="-fPIC"                                   \
+    -D CFLAGS="-fPIC"                                   \
+    -D CXXFLAGS="-fPIC"                                 \
+    -D LDFLAGS=""                                       \
+    -D ENABLE_STATIC:BOOL=ON                            \
+    -D ENABLE_SHARED:BOOL=OFF                           \
+    -D PROCS_INSTALL=8                                  \
+    -D TPL_LIST:STRING="TIMER;ZLIB;HDF5;SILO"           \
+    -D TIMER_URL="${TPL_ROOT}/TimerUtility"             \
+    -D ZLIB_URL="http://zlib.net/zlib-1.2.11.tar.gz"    \
+    -D HDF5_URL="${TPL_ROOT}/hdf5-1.8.12.tar.gz"        \
+        -D BUILD_TYPE=x86_64                            \
+    -D SILO_URL="${TPL_ROOT}/silo-4.10.2.tar.gz"        \
+    ${TPL_BUILDER}
+

sample_scripts/configure_spock_hip

@@ -0,0 +1,30 @@
+module load cmake
+module load llvm-amdgpu
+module load hip
+
+
+export TPL_DIR=/gpfs/alpine/stf006/proj-shared/mbt/spock/install
+
+
+# configure
+rm -rf CMake*
+cmake                                           \
+    -D CMAKE_BUILD_TYPE:STRING=Release          \
+    -D CMAKE_C_COMPILER:PATH=cc                 \
+    -D CMAKE_CXX_COMPILER:PATH=CC               \
+    -D CMAKE_CXX_STANDARD=14                    \
+    -D USE_HIP=1                                \
+        -D LINK_LIBRARIES=${HIP_PATH}/lib/libamdhip64.so \
+    -D USE_CUDA=0                               \
+        -D CMAKE_CUDA_FLAGS="-arch sm_70 -Xptxas=-v -Xptxas -dlcm=cg -lineinfo" \
+        -D CMAKE_CUDA_HOST_COMPILER="gcc"       \
+    -D USE_MPI=0                                \
+    -D USE_HDF5=1                               \
+        -D HDF5_DIRECTORY="${TPL_DIR}/hdf5"     \
+     -D USE_SILO=0                              \
+        -D SILO_DIRECTORY="${TPL_DIR}/silo"     \
+    -D USE_DOXYGEN:BOOL=false		        \
+    -D USE_TIMER=0			        \
+    ~/repos/LBPM-WIA
+
+