From a0b42380a42338349e1f32bfb740ca93acdc7edd Mon Sep 17 00:00:00 2001 From: James McClure Date: Sun, 21 Feb 2021 23:56:10 -0500 Subject: [PATCH 01/32] fixed wide halo bug --- common/WideHalo.cpp | 107 +++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 52 deletions(-) diff --git a/common/WideHalo.cpp b/common/WideHalo.cpp index ee2b2ca5..ca82473f 100644 --- a/common/WideHalo.cpp +++ b/common/WideHalo.cpp @@ -234,59 +234,59 @@ void ScaLBLWideHalo_Communicator::Send(double *data){ //................................................................................... // Send / Recv all the phase indcator field values //................................................................................... - req1[0] = MPI_COMM_SCALBL.Isend(&sendCount_x,1,rank_x,sendtag+0); - req2[0] = MPI_COMM_SCALBL.Irecv(&recvCount_X,1,rank_X,recvtag+0); - req1[1] = MPI_COMM_SCALBL.Isend(&sendCount_X,1,rank_X,sendtag+1); - req2[1] = MPI_COMM_SCALBL.Irecv(&recvCount_x,1,rank_x,recvtag+1); - req1[2] = MPI_COMM_SCALBL.Isend(&sendCount_y,1,rank_y,sendtag+2); - req2[2] = MPI_COMM_SCALBL.Irecv(&recvCount_Y,1,rank_Y,recvtag+2); - req1[3] = MPI_COMM_SCALBL.Isend(&sendCount_Y,1,rank_Y,sendtag+3); - req2[3] = MPI_COMM_SCALBL.Irecv(&recvCount_y,1,rank_y,recvtag+3); - req1[4] = MPI_COMM_SCALBL.Isend(&sendCount_z,1,rank_z,sendtag+4); - req2[4] = MPI_COMM_SCALBL.Irecv(&recvCount_Z,1,rank_Z,recvtag+4); - req1[5] = MPI_COMM_SCALBL.Isend(&sendCount_Z,1,rank_Z,sendtag+5); - req2[5] = MPI_COMM_SCALBL.Irecv(&recvCount_z,1,rank_z,recvtag+5); - req1[6] = MPI_COMM_SCALBL.Isend(&sendCount_xy,1,rank_xy,sendtag+6); - req2[6] = MPI_COMM_SCALBL.Irecv(&recvCount_XY,1,rank_XY,recvtag+6); - req1[7] = MPI_COMM_SCALBL.Isend(&sendCount_XY,1,rank_XY,sendtag+7); - req2[7] = MPI_COMM_SCALBL.Irecv(&recvCount_xy,1,rank_xy,recvtag+7); - req1[8] = MPI_COMM_SCALBL.Isend(&sendCount_Xy,1,rank_Xy,sendtag+8); - req2[8] = MPI_COMM_SCALBL.Irecv(&recvCount_xY,1,rank_xY,recvtag+8); - req1[9] = MPI_COMM_SCALBL.Isend(&sendCount_xY,1,rank_xY,sendtag+9); - req2[9] = MPI_COMM_SCALBL.Irecv(&recvCount_Xy,1,rank_Xy,recvtag+9); - req1[10] = MPI_COMM_SCALBL.Isend(&sendCount_xz,1,rank_xz,sendtag+10); - req2[10] = MPI_COMM_SCALBL.Irecv(&recvCount_XZ,1,rank_XZ,recvtag+10); - req1[11] = MPI_COMM_SCALBL.Isend(&sendCount_XZ,1,rank_XZ,sendtag+11); - req2[11] = MPI_COMM_SCALBL.Irecv(&recvCount_xz,1,rank_xz,recvtag+11); - req1[12] = MPI_COMM_SCALBL.Isend(&sendCount_Xz,1,rank_Xz,sendtag+12); - req2[12] = MPI_COMM_SCALBL.Irecv(&recvCount_xZ,1,rank_xZ,recvtag+12); - req1[13] = MPI_COMM_SCALBL.Isend(&sendCount_xZ,1,rank_xZ,sendtag+13); - req2[13] = MPI_COMM_SCALBL.Irecv(&recvCount_Xz,1,rank_Xz,recvtag+13); - req1[14] = MPI_COMM_SCALBL.Isend(&sendCount_yz,1,rank_yz,sendtag+14); - req2[14] = MPI_COMM_SCALBL.Irecv(&recvCount_YZ,1,rank_YZ,recvtag+14); - req1[15] = MPI_COMM_SCALBL.Isend(&sendCount_YZ,1,rank_YZ,sendtag+15); - req2[15] = MPI_COMM_SCALBL.Irecv(&recvCount_yz,1,rank_yz,recvtag+15); - req1[16] = MPI_COMM_SCALBL.Isend(&sendCount_Yz,1,rank_Yz,sendtag+16); - req2[16] = MPI_COMM_SCALBL.Irecv(&recvCount_yZ,1,rank_yZ,recvtag+16); - req1[17] = MPI_COMM_SCALBL.Isend(&sendCount_yZ,1,rank_yZ,sendtag+17); - req2[17] = MPI_COMM_SCALBL.Irecv(&recvCount_Yz,1,rank_Yz,recvtag+17); + req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x,sendCount_x,rank_x,sendtag+0); + req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X,recvCount_X,rank_X,recvtag+0); + req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X,sendCount_X,rank_X,sendtag+1); + req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x,recvCount_x,rank_x,recvtag+1); + req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y,sendCount_y,rank_y,sendtag+2); + req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y,recvCount_Y,rank_Y,recvtag+2); + req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y,sendCount_Y,rank_Y,sendtag+3); + req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y,recvCount_y,rank_y,recvtag+3); + req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z,sendCount_z,rank_z,sendtag+4); + req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z,recvCount_Z,rank_Z,recvtag+4); + req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z,sendCount_Z,rank_Z,sendtag+5); + req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z,recvCount_z,rank_z,recvtag+5); + req1[6] = MPI_COMM_SCALBL.Isend(sendbuf_xy,sendCount_xy,rank_xy,sendtag+6); + req2[6] = MPI_COMM_SCALBL.Irecv(recvbuf_XY,recvCount_XY,rank_XY,recvtag+6); + req1[7] = MPI_COMM_SCALBL.Isend(sendbuf_XY,sendCount_XY,rank_XY,sendtag+7); + req2[7] = MPI_COMM_SCALBL.Irecv(recvbuf_xy,recvCount_xy,rank_xy,recvtag+7); + req1[8] = MPI_COMM_SCALBL.Isend(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag+8); + req2[8] = MPI_COMM_SCALBL.Irecv(recvbuf_xY,recvCount_xY,rank_xY,recvtag+8); + req1[9] = MPI_COMM_SCALBL.Isend(sendbuf_xY,sendCount_xY,rank_xY,sendtag+9); + req2[9] = MPI_COMM_SCALBL.Irecv(recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag+9); + req1[10] = MPI_COMM_SCALBL.Isend(sendbuf_xz,sendCount_xz,rank_xz,sendtag+10); + req2[10] = MPI_COMM_SCALBL.Irecv(recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag+10); + req1[11] = MPI_COMM_SCALBL.Isend(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag+11); + req2[11] = MPI_COMM_SCALBL.Irecv(recvbuf_xz,recvCount_xz,rank_xz,recvtag+11); + req1[12] = MPI_COMM_SCALBL.Isend(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag+12); + req2[12] = MPI_COMM_SCALBL.Irecv(recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag+12); + req1[13] = MPI_COMM_SCALBL.Isend(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag+13); + req2[13] = MPI_COMM_SCALBL.Irecv(recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag+13); + req1[14] = MPI_COMM_SCALBL.Isend(sendbuf_yz,sendCount_yz,rank_yz,sendtag+14); + req2[14] = MPI_COMM_SCALBL.Irecv(recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag+14); + req1[15] = MPI_COMM_SCALBL.Isend(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag+15); + req2[15] = MPI_COMM_SCALBL.Irecv(recvbuf_yz,recvCount_yz,rank_yz,recvtag+15); + req1[16] = MPI_COMM_SCALBL.Isend(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag+16); + req2[16] = MPI_COMM_SCALBL.Irecv(recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag+16); + req1[17] = MPI_COMM_SCALBL.Isend(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag+17); + req2[17] = MPI_COMM_SCALBL.Irecv(recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag+17); /* Corners */ - req1[18] = MPI_COMM_SCALBL.Isend(&sendCount_xyz,1,rank_xyz,sendtag+18); - req2[18] = MPI_COMM_SCALBL.Irecv(&recvCount_XYZ,1,rank_XYZ,recvtag+18); - req1[19] = MPI_COMM_SCALBL.Isend(&sendCount_XYz,1,rank_XYz,sendtag+19); - req2[19] = MPI_COMM_SCALBL.Irecv(&recvCount_xyZ,1,rank_xyZ,recvtag+19); - req1[20] = MPI_COMM_SCALBL.Isend(&sendCount_Xyz,1,rank_Xyz,sendtag+20); - req2[20] = MPI_COMM_SCALBL.Irecv(&recvCount_xYZ,1,rank_xYZ,recvtag+20); - req1[21] = MPI_COMM_SCALBL.Isend(&sendCount_xYz,1,rank_xYz,sendtag+21); - req2[21] = MPI_COMM_SCALBL.Irecv(&recvCount_XyZ,1,rank_XyZ,recvtag+21); - req1[22] = MPI_COMM_SCALBL.Isend(&sendCount_xyZ,1,rank_xyZ,sendtag+22); - req2[22] = MPI_COMM_SCALBL.Irecv(&recvCount_XYz,1,rank_XYz,recvtag+22); - req1[23] = MPI_COMM_SCALBL.Isend(&sendCount_XYZ,1,rank_XYZ,sendtag+23); - req2[23] = MPI_COMM_SCALBL.Irecv(&recvCount_xyz,1,rank_xyz,recvtag+23); - req1[24] = MPI_COMM_SCALBL.Isend(&sendCount_XyZ,1,rank_XyZ,sendtag+24); - req2[24] = MPI_COMM_SCALBL.Irecv(&recvCount_xYz,1,rank_xYz,recvtag+24); - req1[25] = MPI_COMM_SCALBL.Isend(&sendCount_xYZ,1,rank_xYZ,sendtag+25); - req2[25] = MPI_COMM_SCALBL.Irecv(&recvCount_Xyz,1,rank_Xyz,recvtag+25); + req1[18] = MPI_COMM_SCALBL.Isend(sendbuf_xyz,sendCount_xyz,rank_xyz,sendtag+18); + req2[18] = MPI_COMM_SCALBL.Irecv(recvbuf_XYZ,recvCount_XYZ,rank_XYZ,recvtag+18); + req1[19] = MPI_COMM_SCALBL.Isend(sendbuf_XYz,sendCount_XYz,rank_XYz,sendtag+19); + req2[19] = MPI_COMM_SCALBL.Irecv(recvbuf_xyZ,recvCount_xyZ,rank_xyZ,recvtag+19); + req1[20] = MPI_COMM_SCALBL.Isend(sendbuf_Xyz,sendCount_Xyz,rank_Xyz,sendtag+20); + req2[20] = MPI_COMM_SCALBL.Irecv(recvbuf_xYZ,recvCount_xYZ,rank_xYZ,recvtag+20); + req1[21] = MPI_COMM_SCALBL.Isend(sendbuf_xYz,sendCount_xYz,rank_xYz,sendtag+21); + req2[21] = MPI_COMM_SCALBL.Irecv(recvbuf_XyZ,recvCount_XyZ,rank_XyZ,recvtag+21); + req1[22] = MPI_COMM_SCALBL.Isend(sendbuf_xyZ,sendCount_xyZ,rank_xyZ,sendtag+22); + req2[22] = MPI_COMM_SCALBL.Irecv(recvbuf_XYz,recvCount_XYz,rank_XYz,recvtag+22); + req1[23] = MPI_COMM_SCALBL.Isend(sendbuf_XYZ,sendCount_XYZ,rank_XYZ,sendtag+23); + req2[23] = MPI_COMM_SCALBL.Irecv(recvbuf_xyz,recvCount_xyz,rank_xyz,recvtag+23); + req1[24] = MPI_COMM_SCALBL.Isend(sendbuf_XyZ,sendCount_XyZ,rank_XyZ,sendtag+24); + req2[24] = MPI_COMM_SCALBL.Irecv(recvbuf_xYz,recvCount_xYz,rank_xYz,recvtag+24); + req1[25] = MPI_COMM_SCALBL.Isend(sendbuf_xYZ,sendCount_xYZ,rank_xYZ,sendtag+25); + req2[25] = MPI_COMM_SCALBL.Irecv(recvbuf_Xyz,recvCount_Xyz,rank_Xyz,recvtag+25); //................................................................................... } @@ -302,6 +302,9 @@ void ScaLBLWideHalo_Communicator::Recv(double *data){ Utilities::MPI::waitAll(26,req2); ScaLBL_DeviceBarrier(); //................................................................................... + //printf("Ready to unpack %i to x\n",recvCount_x); + //printf(" print first 10 values...\n"); + //for (int idx=0; idx<10; idx++) printf(" recvBuf[%i]=%f \n",idx,recvbuf_x[idx]); ScaLBL_Scalar_Unpack(dvcRecvList_x, recvCount_x,recvbuf_x, data, Nh); ScaLBL_Scalar_Unpack(dvcRecvList_y, recvCount_y,recvbuf_y, data, Nh); ScaLBL_Scalar_Unpack(dvcRecvList_X, recvCount_X,recvbuf_X, data, Nh); From 991350500f684af0cabc65e8ad724fcbb93b0ce7 Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Thu, 25 Feb 2021 11:38:45 -0500 Subject: [PATCH 02/32] Updating manual configuration of MPI --- cmake/libraries.cmake | 132 ++++++++++++++++-------------------------- cmake/macros.cmake | 4 +- 2 files changed, 53 insertions(+), 83 deletions(-) diff --git a/cmake/libraries.cmake b/cmake/libraries.cmake index 43d2726e..14ff1ac9 100644 --- a/cmake/libraries.cmake +++ b/cmake/libraries.cmake @@ -42,92 +42,62 @@ ENDMACRO() MACRO( CONFIGURE_MPI ) # Determine if we want to use MPI CHECK_ENABLE_FLAG(USE_MPI 1 ) + CHECK_ENABLE_FLAG( USE_MPI 1 ) + MESSAGE("MPIEXEC = ${MPIEXEC}") IF ( USE_MPI ) - # Check if we specified the MPI directory - IF ( MPI_DIRECTORY ) - # Check the provided MPI directory for include files - VERIFY_PATH( "${MPI_DIRECTORY}" ) - IF ( EXISTS "${MPI_DIRECTORY}/include/mpi.h" ) - SET( MPI_INCLUDE_PATH "${MPI_DIRECTORY}/include" ) - ELSEIF ( EXISTS "${MPI_DIRECTORY}/Inc/mpi.h" ) - SET( MPI_INCLUDE_PATH "${MPI_DIRECTORY}/Inc" ) - ELSE() - MESSAGE( FATAL_ERROR "mpi.h not found in ${MPI_DIRECTORY}/include" ) - ENDIF () - INCLUDE_DIRECTORIES ( ${MPI_INCLUDE_PATH} ) - SET ( MPI_INCLUDE ${MPI_INCLUDE_PATH} ) - # Set MPI libraries - IF ( ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" ) - FIND_LIBRARY( MSMPI_LIB NAMES msmpi PATHS "${MPI_DIRECTORY}/Lib/x64" NO_DEFAULT_PATH ) - FIND_LIBRARY( MSMPI_LIB NAMES msmpi PATHS "${MPI_DIRECTORY}/Lib/amd64" NO_DEFAULT_PATH ) - FIND_LIBRARY( MSMPIFEC_LIB NAMES msmpifec PATHS "${MPI_DIRECTORY}/Lib/x64" NO_DEFAULT_PATH ) - FIND_LIBRARY( MSMPIFEC_LIB NAMES msmpifec PATHS "${MPI_DIRECTORY}/Lib/amd64" NO_DEFAULT_PATH ) - FIND_LIBRARY( MSMPIFMC_LIB NAMES msmpifmc PATHS "${MPI_DIRECTORY}/Lib/x64" NO_DEFAULT_PATH ) - FIND_LIBRARY( MSMPIFMC_LIB NAMES msmpifmc PATHS "${MPI_DIRECTORY}/Lib/amd64" NO_DEFAULT_PATH ) - SET( MPI_LIBRARIES ${MSMPI_LIB} ${MSMPIFEC_LIB} ${MSMPIFMC_LIB} ) - ENDIF() - # Set the mpi executable - IF ( MPIEXEC ) - # User specified the MPI command directly, use as is - ELSEIF ( MPIEXEC_CMD ) - # User specified the name of the MPI executable - SET ( MPIEXEC ${MPI_DIRECTORY}/bin/${MPIEXEC_CMD} ) - IF ( NOT EXISTS ${MPIEXEC} ) - MESSAGE( FATAL_ERROR "${MPIEXEC_CMD} not found in ${MPI_DIRECTORY}/bin" ) - ENDIF () - ELSE () - # Search for the MPI executable in the current directory - FIND_PROGRAM( MPIEXEC NAMES mpiexec mpirun lamexec PATHS ${MPI_DIRECTORY}/bin NO_DEFAULT_PATH ) - IF ( NOT MPIEXEC ) - MESSAGE( FATAL_ERROR "Could not locate mpi executable" ) - ENDIF() - ENDIF () - # Set MPI flags - IF ( NOT MPIEXEC_NUMPROC_FLAG ) - SET( MPIEXEC_NUMPROC_FLAG "-np" ) - ENDIF() - ELSEIF ( MPI_COMPILER ) - # The mpi compiler should take care of everything - IF ( MPI_INCLUDE ) - INCLUDE_DIRECTORIES( ${MPI_INCLUDE} ) - ENDIF() + MESSAGE( "Configuring MPI" ) + IF ( MPIEXEC ) + SET( MPIEXEC_EXECUTABLE ${MPIEXEC} ) + ENDIF() + IF ( NOT MPI_SKIP_SEARCH ) + FIND_PACKAGE( MPI ) ELSE() - # Perform the default search for MPI - INCLUDE ( FindMPI ) - IF ( NOT MPI_FOUND ) - MESSAGE( " MPI_INCLUDE = ${MPI_INCLUDE}" ) - MESSAGE( " MPI_LINK_FLAGS = ${MPI_LINK_FLAGS}" ) - MESSAGE( " MPI_LIBRARIES = ${MPI_LIBRARIES}" ) - MESSAGE( FATAL_ERROR "Did not find MPI" ) - ENDIF () - INCLUDE_DIRECTORIES( "${MPI_INCLUDE_PATH}" ) - SET( MPI_INCLUDE "${MPI_INCLUDE_PATH}" ) + # Write mpi test + SET( MPI_TEST_SRC "${CMAKE_CURRENT_BINARY_DIR}/test_mpi.cpp" ) + FILE(WRITE ${MPI_TEST_SRC} "#include \n" ) + FILE(APPEND ${MPI_TEST_SRC} "int main(int argc, char** argv) {\n" ) + FILE(APPEND ${MPI_TEST_SRC} " MPI_Init(NULL, NULL);\n") + FILE(APPEND ${MPI_TEST_SRC} " MPI_Finalize();\n" ) + FILE(APPEND ${MPI_TEST_SRC} "}\n" ) + # Test the compile + IF ( CMAKE_CXX_COMPILER ) + SET( TMP_FLAGS -DINCLUDE_DIRECTORIES=${MPI_CXX_INCLUDE_PATH} ) + TRY_COMPILE( MPI_TEST_CXX ${CMAKE_CURRENT_BINARY_DIR} ${MPI_TEST_SRC} + CMAKE_FLAGS ${TMP_FLAGS} + LINK_OPTIONS ${MPI_CXX_LINK_FLAGS} + LINK_LIBRARIES ${MPI_CXX_LIBRARIES} + OUTPUT_VARIABLE OUT_TXT) + IF ( NOT ${MPI_TEST} ) + MESSAGE( FATAL_ERROR "Skipping MPI search and default compile fails:\n${OUT_TXT}" ) + ENDIF() + SET( MPI_C_FOUND TRUE ) + SET( MPI_CXX_FOUND TRUE ) + SET( MPI_Fortran_FOUND TRUE ) + ENDIF() ENDIF() - # Check if we need to use MPI for serial tests - CHECK_ENABLE_FLAG( USE_MPI_FOR_SERIAL_TESTS 0 ) - # Set defaults if they have not been set - IF ( NOT MPIEXEC ) - SET( MPIEXEC mpirun ) + STRING( STRIP "${MPI_CXX_COMPILE_FLAGS}" MPI_CXX_COMPILE_FLAGS ) + STRING( STRIP "${MPI_CXX_LINK_FLAGS}" MPI_CXX_LINK_FLAGS ) + STRING( STRIP "${MPI_CXX_LIBRARIES}" MPI_CXX_LIBRARIES ) + MESSAGE( " MPI_CXX_FOUND = ${MPI_CXX_FOUND}" ) + MESSAGE( " MPI_CXX_COMPILER = ${MPI_CXX_COMPILER}" ) + MESSAGE( " MPI_CXX_COMPILE_FLAGS = ${MPI_CXX_COMPILE_FLAGS}" ) + MESSAGE( " MPI_CXX_INCLUDE_PATH = ${MPI_CXX_INCLUDE_PATH}" ) + MESSAGE( " MPI_CXX_LINK_FLAGS = ${MPI_CXX_LINK_FLAGS}" ) + MESSAGE( " MPI_CXX_LIBRARIES = ${MPI_CXX_LIBRARIES}" ) + MESSAGE( " MPIEXEC = ${MPIEXEC}" ) + MESSAGE( " MPIEXEC_NUMPROC_FLAG = ${MPIEXEC_NUMPROC_FLAG}" ) + MESSAGE( " MPIEXEC_PREFLAGS = ${MPIEXEC_PREFLAGS}" ) + MESSAGE( " MPIEXEC_POSTFLAGS = ${MPIEXEC_POSTFLAGS}" ) + ADD_DEFINITIONS( -DUSE_MPI ) + INCLUDE_DIRECTORIES( ${MPI_CXX_INCLUDE_PATH} ) + SET( MPI_LIBRARIES ${MPI_CXX_LIBRARIES} ) + SET( MPI_LINK_FLAGS ${MPI_CXX_LINK_FLAGS} ) + IF ( NOT MPI_CXX_FOUND ) + MESSAGE( FATAL_ERROR "MPI not found" ) ENDIF() - IF ( NOT MPIEXEC_NUMPROC_FLAG ) - SET( MPIEXEC_NUMPROC_FLAG "-np" ) + IF ( USE_MPI AND NOT MPIEXEC ) + MESSAGE( FATAL_ERROR "Unable to find MPIEXEC, please set it before continuing" ) ENDIF() - # Set the definitions - ADD_DEFINITIONS( "-DUSE_MPI" ) - MESSAGE( "Using MPI" ) - MESSAGE( " MPIEXEC = ${MPIEXEC}" ) - MESSAGE( " MPIEXEC_NUMPROC_FLAG = ${MPIEXEC_NUMPROC_FLAG}" ) - MESSAGE( " MPI_INCLUDE = ${MPI_INCLUDE}" ) - MESSAGE( " MPI_LINK_FLAGS = ${MPI_LINK_FLAGS}" ) - MESSAGE( " MPI_LIBRARIES = ${MPI_LIBRARIES}" ) - ELSE() - SET( USE_MPI_FOR_SERIAL_TESTS 0 ) - SET( MPIEXEC "" ) - SET( MPIEXEC_NUMPROC_FLAG "" ) - SET( MPI_INCLUDE "" ) - SET( MPI_LINK_FLAGS "" ) - SET( MPI_LIBRARIES "" ) - MESSAGE( "Not using MPI, all parallel tests will be disabled" ) ENDIF() ENDMACRO() diff --git a/cmake/macros.cmake b/cmake/macros.cmake index d1c8dbe7..8030dfa4 100644 --- a/cmake/macros.cmake +++ b/cmake/macros.cmake @@ -681,8 +681,8 @@ MACRO( TARGET_LINK_EXTERNAL_LIBRARIES TARGET_NAME ) FOREACH ( tmp ${BLAS_LAPACK_LIBS} ) TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} ) ENDFOREACH() - FOREACH ( MPI_LIBRARIES ) - TARGET_LINK_LIBRARIES( ${EXE} ${ARGN} ${tmp} ) + FOREACH ( tmp ${MPI_LIBRARIES} ) + TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} ) ENDFOREACH() FOREACH ( tmp ${CMAKE_C_IMPLICIT_LINK_LIBRARIES} ${CMAKE_CXX_IMPLICIT_LINK_LIBRARIES} ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES} ) From 99f1d9b72741aac7776349ecb9515b9ae305c73c Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 25 Feb 2021 20:30:48 -0500 Subject: [PATCH 03/32] add a test routine for mixed gradient --- common/ScaLBL.h | 1 + cpu/FreeLee.cpp | 164 ++++++++++++++++++++++++++++++++++++++++ models/FreeLeeModel.cpp | 148 ++++++++++++++++++++++++++++++++++++ models/FreeLeeModel.h | 3 + tests/CMakeLists.txt | 1 + tests/TestMixedGrad.cpp | 85 +++++++++++++++++++++ 6 files changed, 402 insertions(+) create mode 100644 tests/TestMixedGrad.cpp diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 42c51525..a6a4aeae 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -207,6 +207,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborLis extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, double *Vel, double *Pressure, double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad, int start, int finish, Np); // BOUNDARY CONDITION ROUTINES diff --git a/cpu/FreeLee.cpp b/cpu/FreeLee.cpp index c77436a2..8d5eeb0d 100644 --- a/cpu/FreeLee.cpp +++ b/cpu/FreeLee.cpp @@ -1916,3 +1916,167 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, d Pressure[n] = p; } } + +extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad, int start, int finish, Np){ + + int n,nn,nn2x,ijk; + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double m0,m3,m5,m7; + double mm1,mm2,mm4,mm6,mm8,mm9,mm10,mm11,mm12,mm13,mm14,mm15,mm16,mm17,mm18; + double mm3,mm5,mm7; + //double nx,ny,nz;//normal color gradient + double mgx,mgy,mgz;//mixed gradient reaching secondary neighbor + double phi; + + for (int n=start; nid[i] = Mask->id[i]; + Mask->CommInit(); + Np=Mask->PoreCount(); + //........................................................................... + if (rank==0) printf ("Create ScaLBL_Communicator \n"); + // Create a communicator for the device (will use optimized layout) + // ScaLBL_Communicator ScaLBL_Comm(Mask); // original + //ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + //ScaLBL_Comm_Regular = std::shared_ptr(new ScaLBL_Communicator(Mask)); + ScaLBL_Comm_WideHalo = std::shared_ptr(new ScaLBLWideHalo_Communicator(Mask,2)); + + // create the layout for the LBM + int Npad=(Np/16 + 2)*16; + if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); + Map.resize(Nx,Ny,Nz); Map.fill(-2); + auto neighborList= new int[18*Npad]; + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id.data(),Np,2); + comm.barrier(); + + //........................................................................... + // MAIN VARIABLES ALLOCATED HERE + //........................................................................... + // LBM variables + if (rank==0) printf ("Allocating distributions \n"); + //......................device distributions................................. + dist_mem_size = Np*sizeof(double); + neighborSize=18*(Np*sizeof(int)); + //........................................................................... + //ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); + ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &gqbar, 19*dist_mem_size); + //ScaLBL_AllocateDeviceMemory((void **) &hq, 7*dist_mem_size); + //ScaLBL_AllocateDeviceMemory((void **) &mu_phi, dist_mem_size); + //ScaLBL_AllocateDeviceMemory((void **) &Den, dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Nh); + //ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np); + //........................................................................... + // Update GPU data structures + if (rank==0) printf ("Setting up device map and neighbor list \n"); + fflush(stdout); + int *TmpMap; + TmpMap=new int[Np]; + for (int k=1; kMap(i,j,k); + } + } + } + // check that TmpMap is valid + for (int idx=0; idxLastExterior(); idx++){ + auto n = TmpMap[idx]; + if (n > Nxh*Nyh*Nzh){ + printf("Bad value! idx=%i \n", n); + TmpMap[idx] = Nxh*Nyh*Nzh-1; + } + } + for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ + auto n = TmpMap[idx]; + if ( n > Nxh*Nyh*Nzh ){ + printf("Bad value! idx=%i \n",n); + TmpMap[idx] = Nxh*Nyh*Nzh-1; + } + } + // copy the device map + ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); + // copy the neighbor list + //ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); + comm.barrier(); + + double *phase; + phase = new double[Nh]; + + for (int k=0;kid[n] + int x=i-1; + int y=j-1; + int z=k-1; + if (x<0) x=0; + if (y<0) y=0; + if (z<0) z=0; + if (x>=Nx) x=Nx-1; + if (y>=Ny) y=Ny-1; + if (z>=Nz) z=Nz-1; + int n = z*Nx*Ny+y*Nx+x; + phase[nh]=id[n]; + } + } + } + ScaLBL_CopyToDevice(Phi, phase, Nh*sizeof(double)); + ScaLBL_Comm->Barrier(); + comm.barrier(); + delete [] TmpMap; + delete [] neighborList; + delete [] phase; +} + +void ScaLBL_FreeLeeModel::MGTest(){ + + comm.barrier(); + + ScaLBL_Comm_WideHalo->Send(Phi); + ScaLBL_D3Q9_MGTest(dvcMap,Phi,ColorGrad, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_WideHalo->Send(Phi); + ScaLBL_D3Q9_MGTest(dvcMap,Phi,ColorGrad, 0, ScaLBL_Comm->LastExterior(), Np); + + //check the sum of ColorGrad + double cgx_loc = 0.0; + double cgy_loc = 0.0; + double cgz_loc = 0.0; + double cgx,cgy,cgz; + double *ColorGrad_host; + ColorGrad_host = new double [3*Np]; + ScaLBL_CopyToHost(&ColorGrad_host[0],&ColorGrad[0], 3*Np*sizeof(double)); + for (int i = ScaLBL_Comm->FirstInterior(), iLastInterior(),i++){ + cgx_loc+=ColorGrad_host[0*Np+i]; + cgy_loc+=ColorGrad_host[1*Np+i]; + cgz_loc+=ColorGrad_host[2*Np+i]; + } + for (int i = 0, iLastExterior(),i++){ + cgx_loc+=ColorGrad_host[0*Np+i]; + cgy_loc+=ColorGrad_host[1*Np+i]; + cgz_loc+=ColorGrad_host[2*Np+i]; + } + cgx=Dm->Comm.sumReduce( cgx_loc); + cgy=Dm->Comm.sumReduce( cgy_loc); + cgz=Dm->Comm.sumReduce( cgz_loc); + if (rank==0){ + printf("Sum of all x-component of the mixed gradient = %.2g",cgx); + printf("Sum of all y-component of the mixed gradient = %.2g",cgy); + printf("Sum of all z-component of the mixed gradient = %.2g",cgz); + } + + delete [] ColorGrad_host; +} diff --git a/models/FreeLeeModel.h b/models/FreeLeeModel.h index 1e372f50..23afe39b 100644 --- a/models/FreeLeeModel.h +++ b/models/FreeLeeModel.h @@ -34,6 +34,9 @@ public: void Initialize_SingleFluid(); void Run_SingleFluid(); void WriteDebug_SingleFluid(); + // test utilities + void Create_DummyPhase_MGTest(); + void MGTest(); bool Restart,pBC; int timestep,timestepMax; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8df4e6bd..b32c0b57 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -43,6 +43,7 @@ ADD_LBPM_EXECUTABLE( TestPoissonSolver ) ADD_LBPM_EXECUTABLE( TestIonModel ) ADD_LBPM_EXECUTABLE( TestNernstPlanck ) ADD_LBPM_EXECUTABLE( TestPNP_Stokes ) +ADD_LBPM_EXECUTABLE( TestMixedGrad ) diff --git a/tests/TestMixedGrad.cpp b/tests/TestMixedGrad.cpp new file mode 100644 index 00000000..821d6465 --- /dev/null +++ b/tests/TestMixedGrad.cpp @@ -0,0 +1,85 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "common/Utilities.h" +#include "models/FreeLeeModel.h" + +inline void Initialize_DummyPhaseField(ScaLBL_FreeLeeModel &LeeModel){ + // initialize a bubble + int i,j,k,n; + int rank = LeeModel.Mask->rank(); + int Nx = LeeModel.Mask->Nx; + int Ny = LeeModel.Mask->Ny; + int Nz = LeeModel.Mask->Nz; + if (rank == 0) cout << "Setting up dummy phase field..." << endl; + for (k=0;kid[n]=1; + LeeMOdel.id[n] = LeeModel.Mask->id[n]; + } + } + } +} + + +int main( int argc, char **argv ) +{ + + // Initialize + Utilities::startup( argc, argv ); + + // Load the input database + auto db = std::make_shared( argv[1] ); + + { // Limit scope so variables that contain communicators will free before MPI_Finialize + + Utilities::MPI comm( MPI_COMM_WORLD ); + int rank = comm.getRank(); + int nprocs = comm.getSize(); + + if ( rank == 0 ) { + printf( "********************************************************\n" ); + printf( "Running Mixed Gradient Test \n" ); + printf( "********************************************************\n" ); + } + // Initialize compute device + int device = ScaLBL_SetDevice( rank ); + NULL_USE( device ); + ScaLBL_DeviceBarrier(); + comm.barrier(); + + PROFILE_ENABLE( 1 ); + // PROFILE_ENABLE_TRACE(); + // PROFILE_ENABLE_MEMORY(); + PROFILE_SYNCHRONIZE(); + PROFILE_START( "Main" ); + Utilities::setErrorHandlers(); + + auto filename = argv[1]; + ScaLBL_FreeLeeModel LeeModel( rank, nprocs, comm ); + LeeModel.ReadParams( filename ); + LeeModel.SetDomain(); + Initialize_DummyPhaseField(LeeModel); + LeeModel.Create_DummyPhase_MGTest(); + LeeModel.MGTest() + + PROFILE_STOP( "Main" ); + auto file = db->getWithDefault( "TimerFile", "TestMixedGrad" ); + auto level = db->getWithDefault( "TimerLevel", 1 ); + PROFILE_SAVE( file, level ); + // **************************************************** + + + } // Limit scope so variables that contain communicators will free before MPI_Finialize + + Utilities::shutdown(); + return 0; + +} From 1a393ab26604a8025899af02bd2f9973e28c80a7 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 25 Feb 2021 21:25:29 -0500 Subject: [PATCH 04/32] fix typo and build pass --- common/ScaLBL.h | 2 +- cpu/FreeLee.cpp | 2 +- models/FreeLeeModel.cpp | 18 +++++++++--------- tests/TestMixedGrad.cpp | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index a6a4aeae..cbfdb636 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -207,7 +207,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborLis extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, double *Vel, double *Pressure, double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad, int start, int finish, Np); +extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad,int strideY, int strideZ, int start, int finish, int Np); // BOUNDARY CONDITION ROUTINES diff --git a/cpu/FreeLee.cpp b/cpu/FreeLee.cpp index 8d5eeb0d..8aebdb02 100644 --- a/cpu/FreeLee.cpp +++ b/cpu/FreeLee.cpp @@ -1917,7 +1917,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, d } } -extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad, int start, int finish, Np){ +extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad,int strideY, int strideZ, int start, int finish, int Np){ int n,nn,nn2x,ijk; double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; diff --git a/models/FreeLeeModel.cpp b/models/FreeLeeModel.cpp index 588dfb14..77a485f4 100644 --- a/models/FreeLeeModel.cpp +++ b/models/FreeLeeModel.cpp @@ -1069,7 +1069,7 @@ void ScaLBL_FreeLeeModel::Create_DummyPhase_MGTest(){ if (rank==0) printf ("Create ScaLBL_Communicator \n"); // Create a communicator for the device (will use optimized layout) // ScaLBL_Communicator ScaLBL_Comm(Mask); // original - //ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); //ScaLBL_Comm_Regular = std::shared_ptr(new ScaLBL_Communicator(Mask)); ScaLBL_Comm_WideHalo = std::shared_ptr(new ScaLBLWideHalo_Communicator(Mask,2)); @@ -1174,9 +1174,9 @@ void ScaLBL_FreeLeeModel::MGTest(){ comm.barrier(); ScaLBL_Comm_WideHalo->Send(Phi); - ScaLBL_D3Q9_MGTest(dvcMap,Phi,ColorGrad, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_WideHalo->Send(Phi); - ScaLBL_D3Q9_MGTest(dvcMap,Phi,ColorGrad, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q9_MGTest(dvcMap,Phi,ColorGrad,Nx,Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_WideHalo->Recv(Phi); + ScaLBL_D3Q9_MGTest(dvcMap,Phi,ColorGrad,Nx,Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); //check the sum of ColorGrad double cgx_loc = 0.0; @@ -1186,12 +1186,12 @@ void ScaLBL_FreeLeeModel::MGTest(){ double *ColorGrad_host; ColorGrad_host = new double [3*Np]; ScaLBL_CopyToHost(&ColorGrad_host[0],&ColorGrad[0], 3*Np*sizeof(double)); - for (int i = ScaLBL_Comm->FirstInterior(), iLastInterior(),i++){ + for (int i = ScaLBL_Comm->FirstInterior(); iLastInterior();i++){ cgx_loc+=ColorGrad_host[0*Np+i]; cgy_loc+=ColorGrad_host[1*Np+i]; cgz_loc+=ColorGrad_host[2*Np+i]; } - for (int i = 0, iLastExterior(),i++){ + for (int i = 0; iLastExterior();i++){ cgx_loc+=ColorGrad_host[0*Np+i]; cgy_loc+=ColorGrad_host[1*Np+i]; cgz_loc+=ColorGrad_host[2*Np+i]; @@ -1200,9 +1200,9 @@ void ScaLBL_FreeLeeModel::MGTest(){ cgy=Dm->Comm.sumReduce( cgy_loc); cgz=Dm->Comm.sumReduce( cgz_loc); if (rank==0){ - printf("Sum of all x-component of the mixed gradient = %.2g",cgx); - printf("Sum of all y-component of the mixed gradient = %.2g",cgy); - printf("Sum of all z-component of the mixed gradient = %.2g",cgz); + printf("Sum of all x-component of the mixed gradient = %.2g \n",cgx); + printf("Sum of all y-component of the mixed gradient = %.2g \n",cgy); + printf("Sum of all z-component of the mixed gradient = %.2g \n",cgz); } delete [] ColorGrad_host; diff --git a/tests/TestMixedGrad.cpp b/tests/TestMixedGrad.cpp index 821d6465..dad1ae26 100644 --- a/tests/TestMixedGrad.cpp +++ b/tests/TestMixedGrad.cpp @@ -22,7 +22,7 @@ inline void Initialize_DummyPhaseField(ScaLBL_FreeLeeModel &LeeModel){ for (i=0;iid[n]=1; - LeeMOdel.id[n] = LeeModel.Mask->id[n]; + LeeModel.id[n] = LeeModel.Mask->id[n]; } } } @@ -68,7 +68,7 @@ int main( int argc, char **argv ) LeeModel.SetDomain(); Initialize_DummyPhaseField(LeeModel); LeeModel.Create_DummyPhase_MGTest(); - LeeModel.MGTest() + LeeModel.MGTest(); PROFILE_STOP( "Main" ); auto file = db->getWithDefault( "TimerFile", "TestMixedGrad" ); From 9c043202768a8057e399509e8a3962d40098a544 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 26 Feb 2021 01:18:21 -0500 Subject: [PATCH 05/32] fix error where parameter is used before being loaded --- cpu/FreeLee.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cpu/FreeLee.cpp b/cpu/FreeLee.cpp index 8aebdb02..04ed7c72 100644 --- a/cpu/FreeLee.cpp +++ b/cpu/FreeLee.cpp @@ -236,12 +236,14 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, dou for (int n=start; n Date: Mon, 1 Mar 2021 14:02:29 -0500 Subject: [PATCH 06/32] Updating FindMPI --- README.titan | 1 - cmake/FindMPI.cmake | 362 ------------------------- cmake/ctest_script.cmake | 3 +- cmake/libraries.cmake | 2 - ctest_script.cmake | 2 +- sample_scripts/config_build_eos | 1 - sample_scripts/config_build_rhea | 1 - sample_scripts/config_build_titan | 1 - sample_scripts/config_build_titan_silo | 1 - sample_scripts/config_poplar_hip | 1 - sample_scripts/config_summit_hip | 1 - sample_scripts/config_titan | 1 - sample_scripts/configure_arc_cluster | 1 - sample_scripts/configure_arden | 1 - sample_scripts/configure_basic_cluster | 1 - sample_scripts/configure_blueridge | 1 - sample_scripts/configure_cascades_cpu | 1 - sample_scripts/configure_desktop | 1 - sample_scripts/configure_huckleberry | 1 - sample_scripts/configure_summit | 1 - sample_scripts/configure_titan_jem | 1 - sample_scripts/configure_ubuntu | 1 - sample_scripts/daedalus_config | 1 - sample_scripts/promethius_config | 1 - 24 files changed, 2 insertions(+), 387 deletions(-) delete mode 100644 cmake/FindMPI.cmake diff --git a/README.titan b/README.titan index 8d087700..a3178875 100644 --- a/README.titan +++ b/README.titan @@ -26,7 +26,6 @@ cmake \ -D CMAKE_CXX_COMPILER:PATH=CC \ -D CFLAGS="-DCBUB" \ -D CXXFLAGS="-DCBUB" \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=aprun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D CMAKE_BUILD_TYPE:STRING=Debug \ diff --git a/cmake/FindMPI.cmake b/cmake/FindMPI.cmake deleted file mode 100644 index e1578737..00000000 --- a/cmake/FindMPI.cmake +++ /dev/null @@ -1,362 +0,0 @@ -# - Message Passing Interface (MPI) module. -# -# The Message Passing Interface (MPI) is a library used to write -# high-performance parallel applications that use message passing, and -# is typically deployed on a cluster. MPI is a standard interface -# (defined by the MPI forum) for which many implementations are -# available. All of these implementations have somewhat different -# compilation approaches (different include paths, libraries to link -# against, etc.), and this module tries to smooth out those differences. -# -# This module will set the following variables: -# MPI_FOUND TRUE if we have found MPI -# MPI_COMPILE_FLAGS Compilation flags for MPI programs -# MPI_INCLUDE_PATH Include path(s) for MPI header -# MPI_LINK_FLAGS Linking flags for MPI programs -# MPI_LIBRARY First MPI library to link against (cached) -# MPI_EXTRA_LIBRARY Extra MPI libraries to link against (cached) -# MPI_LIBRARIES All libraries to link MPI programs against -# MPIEXEC Executable for running MPI programs -# MPIEXEC_NUMPROC_FLAG Flag to pass to MPIEXEC before giving it the -# number of processors to run on -# MPIEXEC_PREFLAGS Flags to pass to MPIEXEC directly before the -# executable to run. -# MPIEXEC_POSTFLAGS Flags to pass to MPIEXEC after all other flags. -# -# This module will attempt to auto-detect these settings, first by -# looking for a MPI compiler, which many MPI implementations provide -# as a pass-through to the native compiler to simplify the compilation -# of MPI programs. The MPI compiler is stored in the cache variable -# MPI_COMPILER, and will attempt to look for commonly-named drivers -# mpic++, mpicxx, mpiCC, or mpicc. If the compiler driver is found and -# recognized, it will be used to set all of the module variables. To -# skip this auto-detection, set MPI_LIBRARY and MPI_INCLUDE_PATH in -# the CMake cache. -# -# If no compiler driver is found or the compiler driver is not -# recognized, this module will then search for common include paths -# and library names to try to detect MPI. -# -# If CMake initially finds a different MPI than was intended, and you -# want to use the MPI compiler auto-detection for a different MPI -# implementation, set MPI_COMPILER to the MPI compiler driver you want -# to use (e.g., mpicxx) and then set MPI_LIBRARY to the string -# MPI_LIBRARY-NOTFOUND. When you re-configure, auto-detection of MPI -# will run again with the newly-specified MPI_COMPILER. -# -# When using MPIEXEC to execute MPI applications, you should typically -# use all of the MPIEXEC flags as follows: -# ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} PROCS ${MPIEXEC_PREFLAGS} EXECUTABLE -# ${MPIEXEC_POSTFLAGS} ARGS -# where PROCS is the number of processors on which to execute the program, -# EXECUTABLE is the MPI program, and ARGS are the arguments to pass to the -# MPI program. - -#============================================================================= -# Copyright 2001-2009 Kitware, Inc. -# -# Distributed under the OSI-approved BSD License (the "License"); -# see accompanying file Copyright.txt for details. -# -# This software is distributed WITHOUT ANY WARRANTY; without even the -# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the License for more information. -#============================================================================= -# (To distribute this file outside of CMake, substitute the full -# License text for the above reference.) - -# This module is maintained by David Partyka . - -# A set of directories to search through in addition to the standard system paths -# that find_program will search through. -# Microsoft HPC SDK is automatically added to the system path -# Argonne National Labs MPICH2 sets a registry key that we can use. - -set(_MPI_PACKAGE_DIR - mpi - mpich - openmpi - lib/mpi - lib/mpich - lib/openmpi - "MPICH/SDK" - "Microsoft Compute Cluster Pack" - "Microsoft HPC Pack 2008 R2" - ) - -set(_MPI_PREFIX_PATH) -if(WIN32) - list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH\\SMPD;binary]/..") - list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH2;Path]") -endif() - -foreach(SystemPrefixDir ${CMAKE_SYSTEM_PREFIX_PATH}) - foreach(MpiPackageDir ${_MPI_PREFIX_PATH}) - if(EXISTS ${SystemPrefixDir}/${MpiPackageDir}) - list(APPEND _MPI_PREFIX_PATH "${SystemPrefixDir}/${MpiPackageDir}") - endif() - endforeach(MpiPackageDir) -endforeach(SystemPrefixDir) - -# Most mpi distros have some form of mpiexec which gives us something we can reliably look for. -find_program(MPIEXEC - NAMES mpiexec mpirun lamexec - PATHS ${_MPI_PREFIX_PATH} - PATH_SUFFIXES bin - DOC "Executable for running MPI programs." - ) - -# call get_filename_component twice to remove mpiexec and the directory it exists in (typically bin). -# This gives us a fairly reliable base directory to search for /bin /lib and /include from. -get_filename_component(_MPI_BASE_DIR "${MPIEXEC}" PATH) -get_filename_component(_MPI_BASE_DIR "${_MPI_BASE_DIR}" PATH) - -# If there is an mpi compiler find it and interogate (farther below) it for the include -# and lib dirs otherwise we will continue to search from ${_MPI_BASE_DIR}. -find_program(MPI_COMPILER - NAMES mpic++ mpicxx mpiCC mpicc - HINTS "${_MPI_BASE_DIR}" - PATH_SUFFIXES bin - DOC "MPI compiler. Used only to detect MPI compilation flags.") -mark_as_advanced(MPI_COMPILER) - -set(MPIEXEC_NUMPROC_FLAG "-np" CACHE STRING "Flag used by MPI to specify the number of processes for MPIEXEC; the next option will be the number of processes.") -set(MPIEXEC_PREFLAGS "" CACHE STRING "These flags will be directly before the executable that is being run by MPIEXEC.") -set(MPIEXEC_POSTFLAGS "" CACHE STRING "These flags will come after all flags given to MPIEXEC.") -set(MPIEXEC_MAX_NUMPROCS "2" CACHE STRING "Maximum number of processors available to run MPI applications.") -mark_as_advanced(MPIEXEC MPIEXEC_NUMPROC_FLAG MPIEXEC_PREFLAGS - MPIEXEC_POSTFLAGS MPIEXEC_MAX_NUMPROCS) - -if (MPI_INCLUDE_PATH AND MPI_LIBRARY) - # Do nothing: we already have MPI_INCLUDE_PATH and MPI_LIBRARY in - # the cache, and we don't want to override those settings. -elseif (MPI_COMPILER) - # Check whether the -showme:compile option works. This indicates - # that we have either Open MPI or a newer version of LAM-MPI, and - # implies that -showme:link will also work. - # Note that Windows distros do not have an mpi compiler to interogate. - exec_program(${MPI_COMPILER} - ARGS -showme:compile - OUTPUT_VARIABLE MPI_COMPILE_CMDLINE - RETURN_VALUE MPI_COMPILER_RETURN) - - if (MPI_COMPILER_RETURN EQUAL 0) - # If we appear to have -showme:compile, then we should also have - # -showme:link. Try it. - exec_program(${MPI_COMPILER} - ARGS -showme:link - OUTPUT_VARIABLE MPI_LINK_CMDLINE - RETURN_VALUE MPI_COMPILER_RETURN) - - # Note that we probably have -showme:incdirs and -showme:libdirs - # as well. - set(MPI_COMPILER_MAY_HAVE_INCLIBDIRS TRUE) - endif (MPI_COMPILER_RETURN EQUAL 0) - - if (MPI_COMPILER_RETURN EQUAL 0) - # Do nothing: we have our command lines now - else (MPI_COMPILER_RETURN EQUAL 0) - # Older versions of LAM-MPI have "-showme". Try it. - exec_program(${MPI_COMPILER} - ARGS -showme - OUTPUT_VARIABLE MPI_COMPILE_CMDLINE - RETURN_VALUE MPI_COMPILER_RETURN) - endif (MPI_COMPILER_RETURN EQUAL 0) - - if (MPI_COMPILER_RETURN EQUAL 0) - # Do nothing: we have our command lines now - else (MPI_COMPILER_RETURN EQUAL 0) - # MPICH uses "-show". Try it. - exec_program(${MPI_COMPILER} - ARGS -show - OUTPUT_VARIABLE MPI_COMPILE_CMDLINE - RETURN_VALUE MPI_COMPILER_RETURN) - endif (MPI_COMPILER_RETURN EQUAL 0) - - if (MPI_COMPILER_RETURN EQUAL 0) - # We have our command lines, but we might need to copy - # MPI_COMPILE_CMDLINE into MPI_LINK_CMDLINE, if the underlying - if (NOT MPI_LINK_CMDLINE) - SET(MPI_LINK_CMDLINE ${MPI_COMPILE_CMDLINE}) - endif (NOT MPI_LINK_CMDLINE) - else (MPI_COMPILER_RETURN EQUAL 0) - message(STATUS "Unable to determine MPI from MPI driver ${MPI_COMPILER}") - endif (MPI_COMPILER_RETURN EQUAL 0) -endif (MPI_INCLUDE_PATH AND MPI_LIBRARY) - -if (MPI_INCLUDE_PATH AND MPI_LIBRARY) - # Do nothing: we already have MPI_INCLUDE_PATH and MPI_LIBRARY in - # the cache, and we don't want to override those settings. -elseif (MPI_COMPILE_CMDLINE) - # Extract compile flags from the compile command line. - string(REGEX MATCHALL "(^| )-[Df]([^\" ]+|\"[^\"]+\")" MPI_ALL_COMPILE_FLAGS "${MPI_COMPILE_CMDLINE}") - set(MPI_COMPILE_FLAGS_WORK) - foreach(FLAG ${MPI_ALL_COMPILE_FLAGS}) - if (MPI_COMPILE_FLAGS_WORK) - set(MPI_COMPILE_FLAGS_WORK "${MPI_COMPILE_FLAGS_WORK} ${FLAG}") - else(MPI_COMPILE_FLAGS_WORK) - set(MPI_COMPILE_FLAGS_WORK ${FLAG}) - endif(MPI_COMPILE_FLAGS_WORK) - endforeach(FLAG) - - # Extract include paths from compile command line - string(REGEX MATCHALL "(^| )-I([^\" ]+|\"[^\"]+\")" MPI_ALL_INCLUDE_PATHS "${MPI_COMPILE_CMDLINE}") - set(MPI_INCLUDE_PATH_WORK) - foreach(IPATH ${MPI_ALL_INCLUDE_PATHS}) - string(REGEX REPLACE "^ ?-I" "" IPATH ${IPATH}) - string(REGEX REPLACE "//" "/" IPATH ${IPATH}) - list(APPEND MPI_INCLUDE_PATH_WORK ${IPATH}) - endforeach(IPATH) - - if (NOT MPI_INCLUDE_PATH_WORK) - if (MPI_COMPILER_MAY_HAVE_INCLIBDIRS) - # The compile command line didn't have any include paths on it, - # but we may have -showme:incdirs. Use it. - exec_program(${MPI_COMPILER} - ARGS -showme:incdirs - OUTPUT_VARIABLE MPI_INCLUDE_PATH_WORK - RETURN_VALUE MPI_COMPILER_RETURN) - separate_arguments(MPI_INCLUDE_PATH_WORK) - endif (MPI_COMPILER_MAY_HAVE_INCLIBDIRS) - endif (NOT MPI_INCLUDE_PATH_WORK) - - if (NOT MPI_INCLUDE_PATH_WORK) - # If all else fails, just search for mpi.h in the normal include - # paths. - find_path(MPI_INCLUDE_PATH mpi.h - HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH} - PATH_SUFFIXES include - ) - set(MPI_INCLUDE_PATH_WORK ${MPI_INCLUDE_PATH}) - endif (NOT MPI_INCLUDE_PATH_WORK) - - # Extract linker paths from the link command line - string(REGEX MATCHALL "(^| |-Wl,)-L([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_PATHS "${MPI_LINK_CMDLINE}") - set(MPI_LINK_PATH) - foreach(LPATH ${MPI_ALL_LINK_PATHS}) - string(REGEX REPLACE "^(| |-Wl,)-L" "" LPATH ${LPATH}) - string(REGEX REPLACE "//" "/" LPATH ${LPATH}) - list(APPEND MPI_LINK_PATH ${LPATH}) - endforeach(LPATH) - - if (NOT MPI_LINK_PATH) - if (MPI_COMPILER_MAY_HAVE_INCLIBDIRS) - # The compile command line didn't have any linking paths on it, - # but we may have -showme:libdirs. Use it. - exec_program(${MPI_COMPILER} - ARGS -showme:libdirs - OUTPUT_VARIABLE MPI_LINK_PATH - RETURN_VALUE MPI_COMPILER_RETURN) - separate_arguments(MPI_LINK_PATH) - endif (MPI_COMPILER_MAY_HAVE_INCLIBDIRS) - endif (NOT MPI_LINK_PATH) - - # Extract linker flags from the link command line - string(REGEX MATCHALL "(^| )-Wl,([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_FLAGS "${MPI_LINK_CMDLINE}") - set(MPI_LINK_FLAGS_WORK) - foreach(FLAG ${MPI_ALL_LINK_FLAGS}) - if (MPI_LINK_FLAGS_WORK) - set(MPI_LINK_FLAGS_WORK "${MPI_LINK_FLAGS_WORK} ${FLAG}") - else(MPI_LINK_FLAGS_WORK) - set(MPI_LINK_FLAGS_WORK ${FLAG}) - endif(MPI_LINK_FLAGS_WORK) - endforeach(FLAG) - if ( MPI_LINK_FLAGS_WORK ) - string ( REGEX REPLACE "^ " "" MPI_LINK_FLAGS_WORK ${MPI_LINK_FLAGS_WORK} ) - endif () - - # Extract the set of libraries to link against from the link command - # line - string(REGEX MATCHALL "(^| )-l([^\" ]+|\"[^\"]+\")" MPI_LIBNAMES "${MPI_LINK_CMDLINE}") - - # Determine full path names for all of the libraries that one needs - # to link against in an MPI program - set(MPI_LIBRARIES) - foreach(LIB ${MPI_LIBNAMES}) - string(REGEX REPLACE "^ ?-l" "" LIB ${LIB}) - set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE FILEPATH "Cleared" FORCE) - find_library(MPI_LIB ${LIB} HINTS ${MPI_LINK_PATH}) - if (MPI_LIB) - list(APPEND MPI_LIBRARIES ${MPI_LIB}) - elseif (NOT MPI_FIND_QUIETLY) - message(WARNING "Unable to find MPI library ${LIB}") - endif () - endforeach(LIB) - set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE INTERNAL "Scratch variable for MPI detection" FORCE) - - # Chop MPI_LIBRARIES into the old-style MPI_LIBRARY and - # MPI_EXTRA_LIBRARY. - list(LENGTH MPI_LIBRARIES MPI_NUMLIBS) - list(LENGTH MPI_LIBNAMES MPI_NUMLIBS_EXPECTED) - if (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED) - list(GET MPI_LIBRARIES 0 MPI_LIBRARY_WORK) - set(MPI_LIBRARY ${MPI_LIBRARY_WORK} CACHE FILEPATH "MPI library to link against" FORCE) - else (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED) - set(MPI_LIBRARY "MPI_LIBRARY-NOTFOUND" CACHE FILEPATH "MPI library to link against" FORCE) - endif (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED) - if (MPI_NUMLIBS GREATER 1) - set(MPI_EXTRA_LIBRARY_WORK ${MPI_LIBRARIES}) - list(REMOVE_AT MPI_EXTRA_LIBRARY_WORK 0) - set(MPI_EXTRA_LIBRARY ${MPI_EXTRA_LIBRARY_WORK} CACHE STRING "Extra MPI libraries to link against" FORCE) - else (MPI_NUMLIBS GREATER 1) - set(MPI_EXTRA_LIBRARY "MPI_EXTRA_LIBRARY-NOTFOUND" CACHE STRING "Extra MPI libraries to link against" FORCE) - endif (MPI_NUMLIBS GREATER 1) - - # Set up all of the appropriate cache entries - set(MPI_COMPILE_FLAGS ${MPI_COMPILE_FLAGS_WORK} CACHE STRING "MPI compilation flags" FORCE) - set(MPI_INCLUDE_PATH ${MPI_INCLUDE_PATH_WORK} CACHE STRING "MPI include path" FORCE) - set(MPI_LINK_FLAGS ${MPI_LINK_FLAGS_WORK} CACHE STRING "MPI linking flags" FORCE) -else (MPI_COMPILE_CMDLINE) -# No MPI compiler to interogate so attempt to find everything with find functions. - find_path(MPI_INCLUDE_PATH mpi.h - HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH} - PATH_SUFFIXES include Inc - ) - - # Decide between 32-bit and 64-bit libraries for Microsoft's MPI - if("${CMAKE_SIZEOF_VOID_P}" EQUAL 8) - set(MS_MPI_ARCH_DIR amd64) - else() - set(MS_MPI_ARCH_DIR i386) - endif() - - find_library(MPI_LIBRARY - NAMES mpi mpich msmpi - HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH} - PATH_SUFFIXES lib lib/${MS_MPI_ARCH_DIR} Lib Lib/${MS_MPI_ARCH_DIR} - ) - - find_library(MPI_EXTRA_LIBRARY - NAMES mpi++ - HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH} - PATH_SUFFIXES lib - DOC "Extra MPI libraries to link against.") - - set(MPI_COMPILE_FLAGS "" CACHE STRING "MPI compilation flags") - set(MPI_LINK_FLAGS "" CACHE STRING "MPI linking flags") -endif (MPI_INCLUDE_PATH AND MPI_LIBRARY) - -# Set up extra variables to conform to -if (MPI_EXTRA_LIBRARY) - set(MPI_LIBRARIES ${MPI_LIBRARY} ${MPI_EXTRA_LIBRARY}) -else (MPI_EXTRA_LIBRARY) - set(MPI_LIBRARIES ${MPI_LIBRARY}) -endif (MPI_EXTRA_LIBRARY) - -if (MPI_INCLUDE_PATH AND MPI_LIBRARY) - set(MPI_FOUND TRUE) -else (MPI_INCLUDE_PATH AND MPI_LIBRARY) - set(MPI_FOUND FALSE) -endif (MPI_INCLUDE_PATH AND MPI_LIBRARY) - -#include("${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake") -# handle the QUIETLY and REQUIRED arguments -#find_package_handle_standard_args(MPI DEFAULT_MSG MPI_LIBRARY MPI_INCLUDE_PATH) - -mark_as_advanced(MPI_INCLUDE_PATH MPI_COMPILE_FLAGS MPI_LINK_FLAGS MPI_LIBRARY - MPI_EXTRA_LIBRARY) - -# unset to cleanup namespace -unset(_MPI_PACKAGE_DIR) -unset(_MPI_PREFIX_PATH) -unset(_MPI_BASE_DIR) diff --git a/cmake/ctest_script.cmake b/cmake/ctest_script.cmake index 88bf92fe..e460fed5 100644 --- a/cmake/ctest_script.cmake +++ b/cmake/ctest_script.cmake @@ -32,7 +32,6 @@ SET( CMAKE_MAKE_PROGRAM $ENV{CMAKE_MAKE_PROGRAM} ) SET( CTEST_CMAKE_GENERATOR $ENV{CTEST_CMAKE_GENERATOR} ) SET( LDLIBS $ENV{LDLIBS} ) SET( LDFLAGS $ENV{LDFLAGS} ) -SET( MPI_COMPILER $ENV{MPI_COMPILER} ) SET( MPI_DIRECTORY $ENV{MPI_DIRECTORY} ) SET( MPI_INCLUDE $ENV{MPI_INCLUDE} ) SET( MPI_LINK_FLAGS $ENV{MPI_LINK_FLAGS} ) @@ -198,7 +197,7 @@ SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_C_FLAGS='${CFLAGS}';-DCMAKE_CXX_FLA SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DLDFLAGS:STRING='${FLAGS}';-DLDLIBS:STRING='${LDLIBS}'" ) SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DENABLE_GCOV:BOOL=${ENABLE_GCOV}" ) IF ( USE_MPI ) - SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPI_COMPILER:BOOL=true;-DMPIEXEC=${MPIEXEC}") + SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPIEXEC=${MPIEXEC}") IF ( NOT USE_VALGRIND ) SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_MPI_FOR_SERIAL_TESTS:BOOL=true") ENDIF() diff --git a/cmake/libraries.cmake b/cmake/libraries.cmake index 14ff1ac9..f899c289 100644 --- a/cmake/libraries.cmake +++ b/cmake/libraries.cmake @@ -41,9 +41,7 @@ ENDMACRO() # Macro to find and configure the MPI libraries MACRO( CONFIGURE_MPI ) # Determine if we want to use MPI - CHECK_ENABLE_FLAG(USE_MPI 1 ) CHECK_ENABLE_FLAG( USE_MPI 1 ) - MESSAGE("MPIEXEC = ${MPIEXEC}") IF ( USE_MPI ) MESSAGE( "Configuring MPI" ) IF ( MPIEXEC ) diff --git a/ctest_script.cmake b/ctest_script.cmake index 57f7db15..626f870d 100644 --- a/ctest_script.cmake +++ b/ctest_script.cmake @@ -164,7 +164,7 @@ SET( CTEST_OPTIONS ) SET( CTEST_OPTIONS "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}" ) SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_C_COMPILER:PATH=${CC};-DCMAKE_C_FLAGS='${C_FLAGS}';" ) SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_CXX_COMPILER:PATH=${CXX};-DCMAKE_CXX_FLAGS='${CXX_FLAGS}'" ) -SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPI_COMPILER:BOOL=true;-DMPIEXEC=${MPIEXEC};-DUSE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=true") +SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPIEXEC=${MPIEXEC};-DUSE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=true") IF ( USE_CUDA ) SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_CUDA:BOOL=true;-DCUDA_NVCC_FLAGS='${CUDA_FLAGS}';-DCUDA_HOST_COMPILER=${CUDA_HOST_COMPILER}" ) ELSE() diff --git a/sample_scripts/config_build_eos b/sample_scripts/config_build_eos index f4d69f26..8c7aeb92 100755 --- a/sample_scripts/config_build_eos +++ b/sample_scripts/config_build_eos @@ -33,7 +33,6 @@ cmake \ -D CMAKE_CXX_STANDARD=14 \ -D USE_TIMER=false \ -D TIMER_DIRECTORY=${HOME}/timerutility/build/opt \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=aprun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D USE_CUDA=0 \ diff --git a/sample_scripts/config_build_rhea b/sample_scripts/config_build_rhea index 0e9b7296..0f5713da 100755 --- a/sample_scripts/config_build_rhea +++ b/sample_scripts/config_build_rhea @@ -27,7 +27,6 @@ cmake \ -D CMAKE_CXX_STD=11 \ -D USE_TIMER=false \ -D TIMER_DIRECTORY=${HOME}/timerutility/build/opt \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=aprun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D USE_CUDA=0 \ diff --git a/sample_scripts/config_build_titan b/sample_scripts/config_build_titan index 18d50e1a..32fd639a 100755 --- a/sample_scripts/config_build_titan +++ b/sample_scripts/config_build_titan @@ -25,7 +25,6 @@ cmake \ -D CMAKE_CUDA_FLAGS="-arch sm_35" \ -D CMAKE_CUDA_HOST_COMPILER="/opt/gcc/6.3.0/bin/gcc" \ -D USE_MPI=1 \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=aprun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D USE_SILO=1 \ diff --git a/sample_scripts/config_build_titan_silo b/sample_scripts/config_build_titan_silo index 22069a6c..0a3df511 100755 --- a/sample_scripts/config_build_titan_silo +++ b/sample_scripts/config_build_titan_silo @@ -20,7 +20,6 @@ cmake \ -D CMAKE_CUDA_FLAGS="-arch sm_35" \ -D CMAKE_CUDA_HOST_COMPILER="/opt/gcc/6.3.0/bin/gcc" \ -D USE_MPI=1 \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=aprun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D USE_NETCDF=0 \ diff --git a/sample_scripts/config_poplar_hip b/sample_scripts/config_poplar_hip index f1c3159c..5628c074 100755 --- a/sample_scripts/config_poplar_hip +++ b/sample_scripts/config_poplar_hip @@ -36,6 +36,5 @@ cmake \ # MPI_THREAD_MULTIPLE=1 MV2_USE_RDMA_CM=0 MV2_USE_RDMA_CM= MV2_NUM_HCAS=1 MV2_USE_CUDA=1 MV2_ENABLE_AFFINITY=0 srun -n 2 -N 1 --cpu-bind=v -c 1 ./test_MPI -# -D MPI_COMPILER:BOOL=TRUE \ # -D MPIEXEC=mpirun \ # -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ diff --git a/sample_scripts/config_summit_hip b/sample_scripts/config_summit_hip index 23d3d919..b921b14a 100755 --- a/sample_scripts/config_summit_hip +++ b/sample_scripts/config_summit_hip @@ -21,7 +21,6 @@ cmake \ -D HIP_NVCC_OPTIONS="-arch sm_70" \ -D LINK_LIBRARIES="/sw/summit/cuda/9.2.148/lib64/libcudart.so" \ -D USE_MPI=1 \ - -D MPI_COMPILER:BOOL=TRUE \ -D USE_NETCDF=0 \ -D USE_SILO=1 \ -D SILO_DIRECTORY=${TPL_DIR}/silo \ diff --git a/sample_scripts/config_titan b/sample_scripts/config_titan index 8493d58b..f1b02507 100755 --- a/sample_scripts/config_titan +++ b/sample_scripts/config_titan @@ -9,7 +9,6 @@ cmake \ -D CMAKE_CUDA_FLAGS="-arch sm_35" \ -D CMAKE_CUDA_HOST_COMPILER="/usr/bin/gcc" \ -D USE_MPI=1 \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=aprun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ ${HOME}/LBPM-WIA diff --git a/sample_scripts/configure_arc_cluster b/sample_scripts/configure_arc_cluster index f4124cf6..e97553dd 100755 --- a/sample_scripts/configure_arc_cluster +++ b/sample_scripts/configure_arc_cluster @@ -10,7 +10,6 @@ cmake \ -D CMAKE_CXX_COMPILER:PATH=mpicxx \ -D CMAKE_C_FLAGS="" \ -D CMAKE_CXX_FLAGS="" \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=mpirun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D CMAKE_BUILD_TYPE:STRING=Release \ diff --git a/sample_scripts/configure_arden b/sample_scripts/configure_arden index 057732a9..3d0759aa 100755 --- a/sample_scripts/configure_arden +++ b/sample_scripts/configure_arden @@ -5,7 +5,6 @@ cmake -D CMAKE_C_COMPILER:PATH=/opt/arden/openmpi/3.1.2/bin/mpicc \ -D CMAKE_C_FLAGS="-O3 -fPIC" \ -D CMAKE_CXX_FLAGS="-O3 -fPIC " \ -D CMAKE_CXX_STANDARD=14 \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=mpirun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D CMAKE_BUILD_TYPE:STRING=Release \ diff --git a/sample_scripts/configure_basic_cluster b/sample_scripts/configure_basic_cluster index 667ebcad..7a861974 100755 --- a/sample_scripts/configure_basic_cluster +++ b/sample_scripts/configure_basic_cluster @@ -8,7 +8,6 @@ cmake \ -D CMAKE_CXX_COMPILER:PATH=mpicxx \ -D CMAKE_C_FLAGS="" \ -D CMAKE_CXX_FLAGS="" \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=mpirun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D CMAKE_BUILD_TYPE:STRING=Release \ diff --git a/sample_scripts/configure_blueridge b/sample_scripts/configure_blueridge index 667ebcad..7a861974 100755 --- a/sample_scripts/configure_blueridge +++ b/sample_scripts/configure_blueridge @@ -8,7 +8,6 @@ cmake \ -D CMAKE_CXX_COMPILER:PATH=mpicxx \ -D CMAKE_C_FLAGS="" \ -D CMAKE_CXX_FLAGS="" \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=mpirun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D CMAKE_BUILD_TYPE:STRING=Release \ diff --git a/sample_scripts/configure_cascades_cpu b/sample_scripts/configure_cascades_cpu index f6c77004..a6b0e203 100755 --- a/sample_scripts/configure_cascades_cpu +++ b/sample_scripts/configure_cascades_cpu @@ -11,7 +11,6 @@ cmake \ -D CMAKE_CXX_COMPILER:PATH=mpicxx \ -D CMAKE_C_FLAGS="-fPIC" \ -D CMAKE_CXX_FLAGS="-fPIC" \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=mpirun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D CMAKE_BUILD_TYPE:STRING=Release \ diff --git a/sample_scripts/configure_desktop b/sample_scripts/configure_desktop index 1e717c98..38f917ad 100755 --- a/sample_scripts/configure_desktop +++ b/sample_scripts/configure_desktop @@ -7,7 +7,6 @@ cmake \ -D CMAKE_C_FLAGS="-g " \ -D CMAKE_CXX_FLAGS="-g " \ -D CMAKE_CXX_STANDARD=14 \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=mpirun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D CMAKE_BUILD_TYPE:STRING=Release \ diff --git a/sample_scripts/configure_huckleberry b/sample_scripts/configure_huckleberry index 5ca6bb43..abe134b0 100755 --- a/sample_scripts/configure_huckleberry +++ b/sample_scripts/configure_huckleberry @@ -12,7 +12,6 @@ cmake \ -D CMAKE_CXX_COMPILER:PATH=mpicxx \ -D CMAKE_C_FLAGS="-fPIC" \ -D CMAKE_CXX_FLAGS="-fPIC" \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=mpirun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D CMAKE_BUILD_TYPE:STRING=Release \ diff --git a/sample_scripts/configure_summit b/sample_scripts/configure_summit index bcd8a221..af761468 100755 --- a/sample_scripts/configure_summit +++ b/sample_scripts/configure_summit @@ -20,7 +20,6 @@ cmake \ -D CMAKE_CUDA_FLAGS="-arch sm_70 -Xptxas=-v -Xptxas -dlcm=cg -lineinfo" \ -D CMAKE_CUDA_HOST_COMPILER="/sw/summit/gcc/6.4.0/bin/gcc" \ -D USE_MPI=1 \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=mpirun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D USE_HDF5=1 \ diff --git a/sample_scripts/configure_titan_jem b/sample_scripts/configure_titan_jem index 8375ad87..a3b730cd 100755 --- a/sample_scripts/configure_titan_jem +++ b/sample_scripts/configure_titan_jem @@ -15,7 +15,6 @@ cmake \ -D CMAKE_C_COMPILER:PATH=cc \ -D CMAKE_CXX_COMPILER:PATH=CC \ -D CMAKE_CXX_COMPILER:PATH=CC \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=aprun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D CMAKE_BUILD_TYPE:STRING=Debug \ diff --git a/sample_scripts/configure_ubuntu b/sample_scripts/configure_ubuntu index 516925d0..cccb112c 100755 --- a/sample_scripts/configure_ubuntu +++ b/sample_scripts/configure_ubuntu @@ -5,7 +5,6 @@ cmake -D CMAKE_C_COMPILER:PATH=/opt/arden/openmpi/3.1.2/bin/mpicc \ -D CMAKE_C_FLAGS="-O3 -fPIC" \ -D CMAKE_CXX_FLAGS="-O3 -fPIC " \ -D CMAKE_CXX_STANDARD=14 \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=mpirun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D CMAKE_BUILD_TYPE:STRING=Release \ diff --git a/sample_scripts/daedalus_config b/sample_scripts/daedalus_config index f14f3627..f2855433 100755 --- a/sample_scripts/daedalus_config +++ b/sample_scripts/daedalus_config @@ -12,7 +12,6 @@ i -D CMAKE_CXX_COMPILER:PATH=/home/christopher/openmpi/install_dir/bin/mpicxx -D USE_DOXYGEN=false \ # -D CMAKE_C_FLAGS="-std=gnu++11 -w" \ # -D CMAKE_CXX_FLAGS="-std=gnu++11 -w" \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=/home/christopher/openmpi/install_dir/bin/mpirun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D CMAKE_BUILD_TYPE:STRING=Release \ diff --git a/sample_scripts/promethius_config b/sample_scripts/promethius_config index ae26ed94..49f42670 100755 --- a/sample_scripts/promethius_config +++ b/sample_scripts/promethius_config @@ -7,7 +7,6 @@ cmake \ -D CMAKE_C_FLAGS="-g " \ -D CMAKE_CXX_FLAGS="-g -Wno-deprecated-declarations" \ -D CXX_STD=11 \ - -D MPI_COMPILER:BOOL=TRUE \ -D MPIEXEC=mpirun \ -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D CMAKE_BUILD_TYPE:STRING=Debug \ From 3f659a0cc254e1b4466c4eddf913cd9b25b0e64e Mon Sep 17 00:00:00 2001 From: James McClure Date: Thu, 11 Mar 2021 13:55:47 -0500 Subject: [PATCH 07/32] adding mass conservation test --- cpu/FreeLee.cpp | 28 +++++----- tests/CMakeLists.txt | 1 + tests/testGlobalMassFreeLee.cpp | 99 +++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 14 deletions(-) create mode 100644 tests/testGlobalMassFreeLee.cpp diff --git a/cpu/FreeLee.cpp b/cpu/FreeLee.cpp index 04ed7c72..a11f9ba6 100644 --- a/cpu/FreeLee.cpp +++ b/cpu/FreeLee.cpp @@ -778,25 +778,25 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, dou //-------------------------------- BGK collison for phase field ---------------------------------// // q = 0 - hq[n] = h0 - (h0 - 0.3333333333333333*phi)/tauM; + hq[n] = h0 - (h0 - 0.33333333333333333*phi)/tauM; // q = 1 - hq[nr2] = h1 - (h1 - phi*(0.1111111111111111 + 0.5*ux) - (0.5*M*nx*(1 - 4*phi*phi))/W)/tauM; + hq[nr2] = h1 - (h1 - 0.1111111111111111*(phi*(1.0 + 3.0*ux) - (M*nx*(1 - 4*phi*phi))/W)/tauM); // q = 2 - hq[nr1] = h2 - (h2 - phi*(0.1111111111111111 - 0.5*ux) + (0.5*M*nx*(1 - 4*phi*phi))/W)/tauM; + hq[nr1] = h2 - (h2 - 0.1111111111111111*(phi*(1.0 - 3.0*ux) + (M*nx*(1 - 4*phi*phi))/W)/tauM); // q = 3 - hq[nr4] = h3 - (h3 - phi*(0.1111111111111111 + 0.5*uy) - (0.5*M*ny*(1 - 4*phi*phi))/W)/tauM; + hq[nr4] = h3 - (h3 - 0.1111111111111111*(phi*(1.0 + 3.0*uy) - (M*ny*(1 - 4*phi*phi))/W)/tauM); // q = 4 - hq[nr3] = h4 - (h4 - phi*(0.1111111111111111 - 0.5*uy) + (0.5*M*ny*(1 - 4*phi*phi))/W)/tauM; + hq[nr3] = h4 - (h4 - 0.1111111111111111*(phi*(1.0 - 3.0*uy) + (M*ny*(1 - 4*phi*phi))/W)/tauM); // q = 5 - hq[nr6] = h5 - (h5 - phi*(0.1111111111111111 + 0.5*uz) - (0.5*M*nz*(1 - 4*phi*phi))/W)/tauM; + hq[nr6] = h5 - (h5 - 0.1111111111111111*(phi*(1.0 + 3.0*uz) - (M*nz*(1 - 4*phi*phi))/W)/tauM); // q = 6 - hq[nr5] = h6 - (h6 - phi*(0.1111111111111111 - 0.5*uz) + (0.5*M*nz*(1 - 4*phi*phi))/W)/tauM; + hq[nr5] = h6 - (h6 - 0.1111111111111111*(phi*(1.0 - 3.0*uz) + (M*nz*(1 - 4*phi*phi))/W)/tauM); //........................................................................ //Update velocity on device @@ -1369,25 +1369,25 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double //-------------------------------- BGK collison for phase field ---------------------------------// // q = 0 - hq[n] = h0 - (h0 - 0.3333333333333333*phi)/tauM; + hq[n] = h0 - (h0 - 0.33333333333333333*phi)/tauM; // q = 1 - hq[1*Np+n] = h1 - (h1 - phi*(0.1111111111111111 + 0.5*ux) - (0.5*M*nx*(1 - 4*phi*phi))/W)/tauM; + hq[1*Np+n] = h1 - (h1 - 0.1111111111111111*(phi*(1.0 + 3.0*ux) - (M*nx*(1 - 4*phi*phi))/W)/tauM); // q = 2 - hq[2*Np+n] = h2 - (h2 - phi*(0.1111111111111111 - 0.5*ux) + (0.5*M*nx*(1 - 4*phi*phi))/W)/tauM; + hq[2*Np+n] = h2 - (h2 - 0.1111111111111111*(phi*(1.0 - 3.0*ux) + (M*nx*(1 - 4*phi*phi))/W)/tauM); // q = 3 - hq[3*Np+n] = h3 - (h3 - phi*(0.1111111111111111 + 0.5*uy) - (0.5*M*ny*(1 - 4*phi*phi))/W)/tauM; + hq[3*Np+n] = h3 - (h3 - 0.1111111111111111*(phi*(1.0 + 3.0*uy) - (M*ny*(1 - 4*phi*phi))/W)/tauM); // q = 4 - hq[4*Np+n] = h4 - (h4 - phi*(0.1111111111111111 - 0.5*uy) + (0.5*M*ny*(1 - 4*phi*phi))/W)/tauM; + hq[4*Np+n] = h4 - (h4 - 0.1111111111111111*(phi*(1.0 - 3.0*uy) + (M*ny*(1 - 4*phi*phi))/W)/tauM); // q = 5 - hq[5*Np+n] = h5 - (h5 - phi*(0.1111111111111111 + 0.5*uz) - (0.5*M*nz*(1 - 4*phi*phi))/W)/tauM; + hq[5*Np+n] = h5 - (h5 - 0.1111111111111111*(phi*(1.0 + 3.0*uz) - (M*nz*(1 - 4*phi*phi))/W)/tauM); // q = 6 - hq[6*Np+n] = h6 - (h6 - phi*(0.1111111111111111 - 0.5*uz) + (0.5*M*nz*(1 - 4*phi*phi))/W)/tauM; + hq[6*Np+n] = h6 - (h6 - 0.1111111111111111*(phi*(1.0 - 3.0*uz) + (M*nz*(1 - 4*phi*phi))/W)/tauM); //........................................................................ //Update velocity on device diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b32c0b57..d53b28c5 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -62,6 +62,7 @@ ADD_LBPM_TEST( TestMap ) ADD_LBPM_TEST( TestWideHalo ) ADD_LBPM_TEST( TestColorGradDFH ) ADD_LBPM_TEST( TestBubbleDFH ../example/Bubble/input.db) +ADD_LBPM_TEST( testGlobalMassFreeLee ../example/Bubble/input.db) #ADD_LBPM_TEST( TestColorMassBounceback ../example/Bubble/input.db) ADD_LBPM_TEST( TestPressVel ../example/Bubble/input.db) ADD_LBPM_TEST( TestPoiseuille ../example/Piston/poiseuille.db) diff --git a/tests/testGlobalMassFreeLee.cpp b/tests/testGlobalMassFreeLee.cpp new file mode 100644 index 00000000..ba8189f1 --- /dev/null +++ b/tests/testGlobalMassFreeLee.cpp @@ -0,0 +1,99 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "common/Utilities.h" +#include "models/FreeLeeModel.h" + +//******************************************************************* +// Implementation of Free-Energy Two-Phase LBM (Lee model) +//******************************************************************* + +int main( int argc, char **argv ) +{ + + // Initialize + Utilities::startup( argc, argv ); + + // Load the input database + auto db = std::make_shared( argv[1] ); + + { // Limit scope so variables that contain communicators will free before MPI_Finialize + + Utilities::MPI comm( MPI_COMM_WORLD ); + int rank = comm.getRank(); + int nprocs = comm.getSize(); + + if (rank == 0){ + printf("********************************************************\n"); + printf("Running Free Energy Lee LBM \n"); + printf("********************************************************\n"); + } + // Initialize compute device + int device=ScaLBL_SetDevice(rank); + NULL_USE( device ); + ScaLBL_DeviceBarrier(); + comm.barrier(); + + PROFILE_ENABLE(1); + //PROFILE_ENABLE_TRACE(); + //PROFILE_ENABLE_MEMORY(); + PROFILE_SYNCHRONIZE(); + PROFILE_START("Main"); + Utilities::setErrorHandlers(); + + auto filename = argv[1]; + ScaLBL_FreeLeeModel LeeModel( rank,nprocs,comm ); + LeeModel.ReadParams( filename ); + LeeModel.SetDomain(); + LeeModel.ReadInput(); + LeeModel.Create_TwoFluid(); + LeeModel.Initialize_TwoFluid(); + + /* Copy the initial density to test that global mass is conserved */ + int Nx = LeeModel.Dm->Nx; + int Ny = LeeModel.Dm->Ny; + int Nz = LeeModel.Dm->Nz; + DoubleArray DensityInit(Nx,Ny,Nz); + LeeModel.ScaLBL_Comm->RegularLayout(LeeModel.Map,LeeModel.Den,DensityInit); + + LeeModel.Run_TwoFluid(); + + DoubleArray DensityFinal(Nx,Ny,Nz); + LeeModel.ScaLBL_Comm->RegularLayout(LeeModel.Map,LeeModel.Den,DensityFinal); + + DoubleArray DensityChange(Nx,Ny,Nz); + double totalChange=0.0; + for (int k=1; k Date: Thu, 11 Mar 2021 14:15:18 -0500 Subject: [PATCH 08/32] update ubuntu config --- sample_scripts/configure_ubuntu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample_scripts/configure_ubuntu b/sample_scripts/configure_ubuntu index 516925d0..057732a9 100755 --- a/sample_scripts/configure_ubuntu +++ b/sample_scripts/configure_ubuntu @@ -16,7 +16,7 @@ cmake -D CMAKE_C_COMPILER:PATH=/opt/arden/openmpi/3.1.2/bin/mpicc \ -D USE_SILO=1 \ -D SILO_LIB="/opt/arden/silo/4.10.2/lib/libsiloh5.a" \ -D SILO_DIRECTORY="/opt/arden/silo/4.10.2" \ - -D USE_NETCDF=1 \ + -D USE_NETCDF=0 \ -D NETCDF_DIRECTORY="/opt/arden/netcdf/4.6.1" \ -D USE_CUDA=0 \ -D USE_TIMER=0 \ From ec6b3c6a68e93862b6f28dc65d6cc6a52f94a087 Mon Sep 17 00:00:00 2001 From: James McClure Date: Thu, 11 Mar 2021 20:16:02 -0500 Subject: [PATCH 09/32] revert to cs^2=2/9 for D3Q7 --- cpu/FreeLee.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cpu/FreeLee.cpp b/cpu/FreeLee.cpp index a11f9ba6..04ed7c72 100644 --- a/cpu/FreeLee.cpp +++ b/cpu/FreeLee.cpp @@ -778,25 +778,25 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, dou //-------------------------------- BGK collison for phase field ---------------------------------// // q = 0 - hq[n] = h0 - (h0 - 0.33333333333333333*phi)/tauM; + hq[n] = h0 - (h0 - 0.3333333333333333*phi)/tauM; // q = 1 - hq[nr2] = h1 - (h1 - 0.1111111111111111*(phi*(1.0 + 3.0*ux) - (M*nx*(1 - 4*phi*phi))/W)/tauM); + hq[nr2] = h1 - (h1 - phi*(0.1111111111111111 + 0.5*ux) - (0.5*M*nx*(1 - 4*phi*phi))/W)/tauM; // q = 2 - hq[nr1] = h2 - (h2 - 0.1111111111111111*(phi*(1.0 - 3.0*ux) + (M*nx*(1 - 4*phi*phi))/W)/tauM); + hq[nr1] = h2 - (h2 - phi*(0.1111111111111111 - 0.5*ux) + (0.5*M*nx*(1 - 4*phi*phi))/W)/tauM; // q = 3 - hq[nr4] = h3 - (h3 - 0.1111111111111111*(phi*(1.0 + 3.0*uy) - (M*ny*(1 - 4*phi*phi))/W)/tauM); + hq[nr4] = h3 - (h3 - phi*(0.1111111111111111 + 0.5*uy) - (0.5*M*ny*(1 - 4*phi*phi))/W)/tauM; // q = 4 - hq[nr3] = h4 - (h4 - 0.1111111111111111*(phi*(1.0 - 3.0*uy) + (M*ny*(1 - 4*phi*phi))/W)/tauM); + hq[nr3] = h4 - (h4 - phi*(0.1111111111111111 - 0.5*uy) + (0.5*M*ny*(1 - 4*phi*phi))/W)/tauM; // q = 5 - hq[nr6] = h5 - (h5 - 0.1111111111111111*(phi*(1.0 + 3.0*uz) - (M*nz*(1 - 4*phi*phi))/W)/tauM); + hq[nr6] = h5 - (h5 - phi*(0.1111111111111111 + 0.5*uz) - (0.5*M*nz*(1 - 4*phi*phi))/W)/tauM; // q = 6 - hq[nr5] = h6 - (h6 - 0.1111111111111111*(phi*(1.0 - 3.0*uz) + (M*nz*(1 - 4*phi*phi))/W)/tauM); + hq[nr5] = h6 - (h6 - phi*(0.1111111111111111 - 0.5*uz) + (0.5*M*nz*(1 - 4*phi*phi))/W)/tauM; //........................................................................ //Update velocity on device @@ -1369,25 +1369,25 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double //-------------------------------- BGK collison for phase field ---------------------------------// // q = 0 - hq[n] = h0 - (h0 - 0.33333333333333333*phi)/tauM; + hq[n] = h0 - (h0 - 0.3333333333333333*phi)/tauM; // q = 1 - hq[1*Np+n] = h1 - (h1 - 0.1111111111111111*(phi*(1.0 + 3.0*ux) - (M*nx*(1 - 4*phi*phi))/W)/tauM); + hq[1*Np+n] = h1 - (h1 - phi*(0.1111111111111111 + 0.5*ux) - (0.5*M*nx*(1 - 4*phi*phi))/W)/tauM; // q = 2 - hq[2*Np+n] = h2 - (h2 - 0.1111111111111111*(phi*(1.0 - 3.0*ux) + (M*nx*(1 - 4*phi*phi))/W)/tauM); + hq[2*Np+n] = h2 - (h2 - phi*(0.1111111111111111 - 0.5*ux) + (0.5*M*nx*(1 - 4*phi*phi))/W)/tauM; // q = 3 - hq[3*Np+n] = h3 - (h3 - 0.1111111111111111*(phi*(1.0 + 3.0*uy) - (M*ny*(1 - 4*phi*phi))/W)/tauM); + hq[3*Np+n] = h3 - (h3 - phi*(0.1111111111111111 + 0.5*uy) - (0.5*M*ny*(1 - 4*phi*phi))/W)/tauM; // q = 4 - hq[4*Np+n] = h4 - (h4 - 0.1111111111111111*(phi*(1.0 - 3.0*uy) + (M*ny*(1 - 4*phi*phi))/W)/tauM); + hq[4*Np+n] = h4 - (h4 - phi*(0.1111111111111111 - 0.5*uy) + (0.5*M*ny*(1 - 4*phi*phi))/W)/tauM; // q = 5 - hq[5*Np+n] = h5 - (h5 - 0.1111111111111111*(phi*(1.0 + 3.0*uz) - (M*nz*(1 - 4*phi*phi))/W)/tauM); + hq[5*Np+n] = h5 - (h5 - phi*(0.1111111111111111 + 0.5*uz) - (0.5*M*nz*(1 - 4*phi*phi))/W)/tauM; // q = 6 - hq[6*Np+n] = h6 - (h6 - 0.1111111111111111*(phi*(1.0 - 3.0*uz) + (M*nz*(1 - 4*phi*phi))/W)/tauM); + hq[6*Np+n] = h6 - (h6 - phi*(0.1111111111111111 - 0.5*uz) + (0.5*M*nz*(1 - 4*phi*phi))/W)/tauM; //........................................................................ //Update velocity on device From aa04ad4898490ea41540789df5f6a3fa1710ecbe Mon Sep 17 00:00:00 2001 From: James McClure Date: Sun, 14 Mar 2021 13:02:55 -0400 Subject: [PATCH 10/32] fixed wide halo bug with lee model --- models/FreeLeeModel.cpp | 18 +++++++++++------- tests/TestMixedGrad.cpp | 1 + tests/testGlobalMassFreeLee.cpp | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/models/FreeLeeModel.cpp b/models/FreeLeeModel.cpp index 77a485f4..9fba430c 100644 --- a/models/FreeLeeModel.cpp +++ b/models/FreeLeeModel.cpp @@ -776,7 +776,7 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){ ScaLBL_Comm_WideHalo->Send(Phi); ScaLBL_D3Q19_AAodd_FreeLeeModel(NeighborList, dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM, - kappa, beta, W, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm_WideHalo->Recv(Phi); ScaLBL_Comm->RecvD3Q19AA(gqbar); //WRITE INTO OPPOSITE ScaLBL_Comm->Barrier(); @@ -794,13 +794,16 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){ ScaLBL_Comm->D3Q19_Reflection_BC_Z(gqbar); } ScaLBL_D3Q19_AAodd_FreeLeeModel(NeighborList, dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM, - kappa, beta, W, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->Barrier(); + + printf("write debug \n"); + WriteDebug_TwoFluid(); // *************EVEN TIMESTEP************* timestep++; // Compute the Phase indicator field - ScaLBL_Comm->SendD3Q7AA(hq,0); //READ FROM NORMAL + ScaLBL_Comm->SendD3Q7AA(hq,0); //READ FROM NORMA ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(dvcMap, hq, Den, Phi, rhoA, rhoB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(hq,0); //WRITE INTO OPPOSITE ScaLBL_Comm->Barrier(); @@ -815,7 +818,7 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){ } ScaLBL_Comm_WideHalo->Send(Phi); ScaLBL_D3Q19_AAeven_FreeLeeModel(dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM, - kappa, beta, W, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm_WideHalo->Recv(Phi); ScaLBL_Comm->RecvD3Q19AA(gqbar); //WRITE INTO OPPOSITE ScaLBL_Comm->Barrier(); @@ -833,7 +836,7 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){ ScaLBL_Comm->D3Q19_Reflection_BC_Z(gqbar); } ScaLBL_D3Q19_AAeven_FreeLeeModel(dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM, - kappa, beta, W, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->Barrier(); //************************************************************************ PROFILE_STOP("Update"); @@ -906,6 +909,7 @@ void ScaLBL_FreeLeeModel::Run_SingleFluid(){ 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->Barrier(); + // *************EVEN TIMESTEP************* timestep++; //------------------------------------------------------------------------------------------------------------------- @@ -1003,7 +1007,7 @@ void ScaLBL_FreeLeeModel::WriteDebug_TwoFluid(){ fwrite(PhaseField.data(),8,N,VELZ_FILE); fclose(VELZ_FILE); -/* ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField); + ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField); FILE *CGX_FILE; sprintf(LocalRankFilename,"Gradient_X.%05i.raw",rank); CGX_FILE = fopen(LocalRankFilename,"wb"); @@ -1023,7 +1027,7 @@ void ScaLBL_FreeLeeModel::WriteDebug_TwoFluid(){ CGZ_FILE = fopen(LocalRankFilename,"wb"); fwrite(PhaseField.data(),8,N,CGZ_FILE); fclose(CGZ_FILE); -*/ + } void ScaLBL_FreeLeeModel::WriteDebug_SingleFluid(){ diff --git a/tests/TestMixedGrad.cpp b/tests/TestMixedGrad.cpp index dad1ae26..7fc3373c 100644 --- a/tests/TestMixedGrad.cpp +++ b/tests/TestMixedGrad.cpp @@ -69,6 +69,7 @@ int main( int argc, char **argv ) Initialize_DummyPhaseField(LeeModel); LeeModel.Create_DummyPhase_MGTest(); LeeModel.MGTest(); + LeeModel.WriteDebug_TwoFluid(); PROFILE_STOP( "Main" ); auto file = db->getWithDefault( "TimerFile", "TestMixedGrad" ); diff --git a/tests/testGlobalMassFreeLee.cpp b/tests/testGlobalMassFreeLee.cpp index ba8189f1..bd54f106 100644 --- a/tests/testGlobalMassFreeLee.cpp +++ b/tests/testGlobalMassFreeLee.cpp @@ -86,7 +86,7 @@ int main( int argc, char **argv ) fwrite(DensityChange.data(),8,Nx*Ny*Nz,OUTFILE); fclose(OUTFILE); - LeeModel.WriteDebug_TwoFluid(); + //LeeModel.WriteDebug_TwoFluid(); PROFILE_STOP("Main"); // **************************************************** From b828f6e5e65c7dd1523f8fc8be3f1c1c00a23eb2 Mon Sep 17 00:00:00 2001 From: James McClure Date: Mon, 15 Mar 2021 12:58:24 -0400 Subject: [PATCH 11/32] updating mixed gradient test --- models/FreeLeeModel.cpp | 10 +++++----- tests/TestMixedGrad.cpp | 42 ++++++++++++++++++++++++++++++++++------- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/models/FreeLeeModel.cpp b/models/FreeLeeModel.cpp index 9fba430c..a4895285 100644 --- a/models/FreeLeeModel.cpp +++ b/models/FreeLeeModel.cpp @@ -10,9 +10,9 @@ color lattice boltzmann model #include ScaLBL_FreeLeeModel::ScaLBL_FreeLeeModel(int RANK, int NP, const Utilities::MPI& COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauM(0),rhoA(0),rhoB(0),W(0),gamma(0),kappa(0),beta(0), +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(2),tauA(1.0),tauB(1.0),tauM(1.0),rhoA(1.0),rhoB(1.0),W(5.0),gamma(0.001),kappa(0.0075),beta(0.0024), Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0), -tau(0),rho0(0), +tau(1.0),rho0(1.0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { @@ -797,7 +797,7 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){ kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->Barrier(); - printf("write debug \n"); + printf("write debug strideY=%i strideZ = %i \n",Nxh, Nxh*Nyh); WriteDebug_TwoFluid(); // *************EVEN TIMESTEP************* @@ -1178,9 +1178,9 @@ void ScaLBL_FreeLeeModel::MGTest(){ comm.barrier(); ScaLBL_Comm_WideHalo->Send(Phi); - ScaLBL_D3Q9_MGTest(dvcMap,Phi,ColorGrad,Nx,Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q9_MGTest(dvcMap,Phi,ColorGrad,Nxh,Nxh*Nyh, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm_WideHalo->Recv(Phi); - ScaLBL_D3Q9_MGTest(dvcMap,Phi,ColorGrad,Nx,Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q9_MGTest(dvcMap,Phi,ColorGrad,Nxh,Nxh*Nyh, 0, ScaLBL_Comm->LastExterior(), Np); //check the sum of ColorGrad double cgx_loc = 0.0; diff --git a/tests/TestMixedGrad.cpp b/tests/TestMixedGrad.cpp index 7fc3373c..bbf686ea 100644 --- a/tests/TestMixedGrad.cpp +++ b/tests/TestMixedGrad.cpp @@ -9,14 +9,15 @@ #include "common/Utilities.h" #include "models/FreeLeeModel.h" -inline void Initialize_DummyPhaseField(ScaLBL_FreeLeeModel &LeeModel){ +inline void Initialize_Mask(ScaLBL_FreeLeeModel &LeeModel){ // initialize a bubble int i,j,k,n; int rank = LeeModel.Mask->rank(); int Nx = LeeModel.Mask->Nx; int Ny = LeeModel.Mask->Ny; int Nz = LeeModel.Mask->Nz; - if (rank == 0) cout << "Setting up dummy phase field..." << endl; + if (rank == 0) printf(" initialize mask...\n"); + for (k=0;krank(); + int Nx = LeeModel.Mask->Nx; + int Ny = LeeModel.Mask->Ny; + int Nz = LeeModel.Mask->Nz; + if (rank == 0) printf("Setting up dummy phase field with gradient {x,y,z} = {%f , %f , %f}...\n",ax,ay,az); + + double * Dummy; + int Nh = (Nx+2)*(Ny+2)*(Nz+2); + Dummy = new double [(Nx+2)*(Ny+2)*(Nz+2)]; + for (k=0;kid[n]=1; + LeeModel.id[n] = LeeModel.Mask->id[n]; + int nh = (k+1)*(Nx+2)*(Ny+2) + (j+1)*(Nx+2) + i+1; + Dummy[nh] = ax*double(i) + ay*double(j) + az*double(k); + } + } + } + ScaLBL_CopyToDevice(LeeModel.Phi, Dummy, sizeof(double)*Nh); + + LeeModel.MGTest(); +} int main( int argc, char **argv ) { @@ -66,14 +94,14 @@ int main( int argc, char **argv ) ScaLBL_FreeLeeModel LeeModel( rank, nprocs, comm ); LeeModel.ReadParams( filename ); LeeModel.SetDomain(); - Initialize_DummyPhaseField(LeeModel); - LeeModel.Create_DummyPhase_MGTest(); - LeeModel.MGTest(); + Initialize_Mask(LeeModel); + //LeeModel.Create_DummyPhase_MGTest(); + LeeModel.Create_TwoFluid(); + + Initialize_DummyPhaseField(LeeModel,1.0, 2.0, 3.0); LeeModel.WriteDebug_TwoFluid(); PROFILE_STOP( "Main" ); - auto file = db->getWithDefault( "TimerFile", "TestMixedGrad" ); - auto level = db->getWithDefault( "TimerLevel", 1 ); PROFILE_SAVE( file, level ); // **************************************************** From 15dbe5f053b8c9af031879feddded2d273861a40 Mon Sep 17 00:00:00 2001 From: James McClure Date: Mon, 15 Mar 2021 17:31:14 -0400 Subject: [PATCH 12/32] update ubuntu configure script --- sample_scripts/configure_ubuntu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample_scripts/configure_ubuntu b/sample_scripts/configure_ubuntu index cccb112c..fab18662 100755 --- a/sample_scripts/configure_ubuntu +++ b/sample_scripts/configure_ubuntu @@ -15,7 +15,7 @@ cmake -D CMAKE_C_COMPILER:PATH=/opt/arden/openmpi/3.1.2/bin/mpicc \ -D USE_SILO=1 \ -D SILO_LIB="/opt/arden/silo/4.10.2/lib/libsiloh5.a" \ -D SILO_DIRECTORY="/opt/arden/silo/4.10.2" \ - -D USE_NETCDF=1 \ + -D USE_NETCDF=0 \ -D NETCDF_DIRECTORY="/opt/arden/netcdf/4.6.1" \ -D USE_CUDA=0 \ -D USE_TIMER=0 \ From 6d59317919e8ff240247af3835de70b8922d5c3b Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Tue, 16 Mar 2021 17:21:52 -0400 Subject: [PATCH 13/32] Cleaning up TestWriter --- IO/Reader.cpp | 36 ++++++- IO/Reader.h | 23 ++++- IO/Writer.cpp | 2 +- tests/TestWriter.cpp | 231 ++++++++++++++++++++++--------------------- tests/convertIO.cpp | 2 +- 5 files changed, 177 insertions(+), 117 deletions(-) diff --git a/IO/Reader.cpp b/IO/Reader.cpp index dedb9b82..fe10eae9 100644 --- a/IO/Reader.cpp +++ b/IO/Reader.cpp @@ -37,10 +37,19 @@ std::string IO::getPath( const std::string& filename ) } -// List the timesteps in the given directors (dumps.LBPM) -std::vector IO::readTimesteps( const std::string& filename ) +// List the timesteps in the given directory (dumps.LBPM) +std::vector IO::readTimesteps( const std::string& path, const std::string& format ) { + // Get the name of the summary filename + std::string filename = path + "/"; + if ( format=="old" || format=="new" ) + filename += "summary.LBM"; + else if ( format=="silo" ) + filename += "LBM.visit"; + else + ERROR( "Unknown format: " + format ); PROFILE_START("readTimesteps"); + // Read the data FILE *fid= fopen(filename.c_str(),"rb"); if ( fid==NULL ) ERROR("Error opening file"); @@ -59,6 +68,29 @@ std::vector IO::readTimesteps( const std::string& filename ) fclose(fid); PROFILE_STOP("readTimesteps"); return timesteps; + return timesteps; +} + + +// Read the data for the given timestep +std::vector IO::readData( const std::string& path, const std::string& timestep, const Utilities::MPI &comm ) +{ + // Get the mesh databases + auto db = IO::getMeshList( path, timestep ); + // Create the data + std::vector data( db .size() ); + for ( size_t i=0; i readTimesteps( const std::string& filename ); +/*! + * @brief Read the timestep list + * @details This function reads the timestep list from the summary file. + * @param[in] path The path to use for reading + * @param[in] format The data format to use: + * old - Old mesh format (provided for backward compatibility) + * new - New format, 1 file/process + * silo - Silo + * @return append Append any existing data (default is false) + */ +std::vector readTimesteps( const std::string& path, const std::string& format ); + + +/*! + * @brief Read the data for the timestep + * @details This function reads the mesh and variable data provided for the given timestep. + * Note: this function requires that the number of ranks of the comm match the number of ranks in the meshes + * @param[in] path The path to use for reading + * @param[in] timestep The timestep iteration + */ +std::vector readData( const std::string& path, const std::string& timestep, const Utilities::MPI &comm = MPI_COMM_WORLD ); //! Read the list of mesh databases for the given timestep diff --git a/IO/Writer.cpp b/IO/Writer.cpp index 61c333af..7414d5a1 100644 --- a/IO/Writer.cpp +++ b/IO/Writer.cpp @@ -76,7 +76,7 @@ static std::vector writeMeshesOrigFormat( const std::vectorname ); } diff --git a/tests/TestWriter.cpp b/tests/TestWriter.cpp index 97774c55..4030930c 100644 --- a/tests/TestWriter.cpp +++ b/tests/TestWriter.cpp @@ -31,13 +31,97 @@ inline double distance( const Point& p ) } +bool checkMesh( const std::vector& meshData, const std::string& format, std::shared_ptr mesh ) +{ + + // Get direct access to the meshes used to test the reader + const auto pointmesh = dynamic_cast( meshData[0].mesh.get() ); + const auto trimesh = dynamic_cast( meshData[1].mesh.get() ); + const auto trilist = dynamic_cast( meshData[2].mesh.get() ); + const auto domain = dynamic_cast( meshData[3].mesh.get() ); + const size_t N_tri = trimesh->A.size(); + if ( mesh->className() == "pointmesh" ) { + // Check the pointmesh + auto pmesh = IO::getPointList(mesh); + if ( pmesh.get()==NULL ) + return false; + if ( pmesh->points.size() != pointmesh->points.size() ) + return false; + } + if ( mesh->className() == "trimesh" || mesh->className() == "trilist" ) { + // Check the trimesh/trilist + auto mesh1 = IO::getTriMesh(mesh); + auto mesh2 = IO::getTriList(mesh); + if ( mesh1.get()==NULL || mesh2.get()==NULL ) + return false; + if ( mesh1->A.size()!=N_tri || mesh1->B.size()!=N_tri || mesh1->C.size()!=N_tri || + mesh2->A.size()!=N_tri || mesh2->B.size()!=N_tri || mesh2->C.size()!=N_tri ) + return false; + const std::vector& P1 = mesh1->vertices->points; + const std::vector& A1 = mesh1->A; + const std::vector& B1 = mesh1->B; + const std::vector& C1 = mesh1->C; + const std::vector& A2 = mesh2->A; + const std::vector& B2 = mesh2->B; + const std::vector& C2 = mesh2->C; + const std::vector& A = trilist->A; + const std::vector& B = trilist->B; + const std::vector& C = trilist->C; + for (size_t i=0; iclassName() == "domain" && format!="old" ) { + // Check the domain mesh + const IO::DomainMesh& mesh1 = *std::dynamic_pointer_cast(mesh); + if ( mesh1.nprocx!=domain->nprocx || mesh1.nprocy!=domain->nprocy || mesh1.nprocz!=domain->nprocz ) + return false; + if ( mesh1.nx!=domain->nx || mesh1.ny!=domain->ny || mesh1.nz!=domain->nz ) + return false; + if ( mesh1.Lx!=domain->Lx || mesh1.Ly!=domain->Ly || mesh1.Lz!=domain->Lz ) + return false; + } + return true; +} + + +bool checkVar( const std::string& format, std::shared_ptr mesh, + std::shared_ptr variable1, std::shared_ptr variable2 ) +{ + if ( format=="new" ) + IO::reformatVariable( *mesh, *variable2 ); + bool pass = true; + const IO::Variable& var1 = *variable1; + const IO::Variable& var2 = *variable2; + pass = var1.name == var2.name; + pass = pass && var1.dim == var2.dim; + pass = pass && var1.type == var2.type; + pass = pass && var1.data.length() == var2.data.length(); + if ( pass ) { + for (size_t m=0; m& meshData, UnitTest& ut ) { + PROFILE_SCOPED( path, 0, timer ); + Utilities::MPI comm( MPI_COMM_WORLD ); int nprocs = comm.getSize(); comm.barrier(); + + // Set the path for the writer + std::string path = "test_" + format; + + // Get the format std::string format2 = format; auto precision = IO::DataType::Double; @@ -49,6 +133,7 @@ void testWriter( const std::string& format, std::vector& mes precision = IO::DataType::Float; } + // Set the precision for the variables for ( auto& data : meshData ) { data.precision = precision; @@ -57,123 +142,59 @@ void testWriter( const std::string& format, std::vector& mes } // Write the data - PROFILE_START(format+"-write"); - IO::initialize( "test_"+format, format2, false ); + IO::initialize( path, format2, false ); IO::writeData( 0, meshData, comm ); IO::writeData( 3, meshData, comm ); comm.barrier(); - PROFILE_STOP(format+"-write"); - // Get the summary name for reading - std::string path = "test_" + format; - std::string summary_name; - if ( format=="old" || format=="new" ) - summary_name = "summary.LBM"; - else if ( format=="silo-float" || format=="silo-double" ) - summary_name = "LBM.visit"; - else - ERROR("Unknown format"); - // Get direct access to the meshes used to test the reader - const auto pointmesh = dynamic_cast( meshData[0].mesh.get() ); - const auto trimesh = dynamic_cast( meshData[1].mesh.get() ); - const auto trilist = dynamic_cast( meshData[2].mesh.get() ); - const auto domain = dynamic_cast( meshData[3].mesh.get() ); - const size_t N_tri = trimesh->A.size(); - - // Get a list of the timesteps - PROFILE_START(format+"-read-timesteps"); - auto timesteps = IO::readTimesteps( path + "/" + summary_name ); - PROFILE_STOP(format+"-read-timesteps"); + // Get a list of the timesteps + auto timesteps = IO::readTimesteps( path, format2 ); if ( timesteps.size()==2 ) ut.passes(format+": Corrent number of timesteps"); else ut.failure(format+": Incorrent number of timesteps"); - // Check the mesh lists + + // Test the simple read interface + bool pass = true; + for ( const auto& timestep : timesteps ) { + auto data = IO::readData( path, timestep ); + pass = pass && data.size() == meshData.size(); + for ( size_t i=0; ipoints.size() != pointmesh->points.size() ) { - pass = false; - break; - } - } - if ( database.name=="trimesh" || database.name=="trilist" ) { - // Check the trimesh/trilist - auto mesh1 = IO::getTriMesh(mesh); - auto mesh2 = IO::getTriList(mesh); - if ( mesh1.get()==NULL || mesh2.get()==NULL ) { - pass = false; - break; - } - if ( mesh1->A.size()!=N_tri || mesh1->B.size()!=N_tri || mesh1->C.size()!=N_tri || - mesh2->A.size()!=N_tri || mesh2->B.size()!=N_tri || mesh2->C.size()!=N_tri ) - { - pass = false; - break; - } - const std::vector& P1 = mesh1->vertices->points; - const std::vector& A1 = mesh1->A; - const std::vector& B1 = mesh1->B; - const std::vector& C1 = mesh1->C; - const std::vector& A2 = mesh2->A; - const std::vector& B2 = mesh2->B; - const std::vector& C2 = mesh2->C; - const std::vector& A = trilist->A; - const std::vector& B = trilist->B; - const std::vector& C = trilist->C; - for (size_t i=0; i(mesh); - if ( mesh1.nprocx!=domain->nprocx || mesh1.nprocy!=domain->nprocy || mesh1.nprocz!=domain->nprocz ) - pass = false; - if ( mesh1.nx!=domain->nx || mesh1.ny!=domain->ny || mesh1.nz!=domain->nz ) - pass = false; - if ( mesh1.Lx!=domain->Lx || mesh1.Ly!=domain->Ly || mesh1.Lz!=domain->Lz ) - pass = false; + } else { + pass = pass && checkMesh( meshData, format, mesh ); } } if ( pass ) { @@ -185,31 +206,19 @@ void testWriter( const std::string& format, std::vector& mes // Load the variables and check their data if ( format=="old" ) continue; // Old format does not support variables - const IO::MeshDataStruct* mesh0 = NULL; + const IO::MeshDataStruct* mesh0 = nullptr; for (size_t k=0; kvars.size(); v++) { PROFILE_START(format+"-read-getVariable"); auto variable = IO::getVariable(path,timestep,database,k,mesh0->vars[v]->name); - if ( format=="new" ) - IO::reformatVariable( *mesh, *variable ); - PROFILE_STOP(format+"-read-getVariable"); - const IO::Variable& var1 = *mesh0->vars[v]; - const IO::Variable& var2 = *variable; - pass = var1.name == var2.name; - pass = pass && var1.dim == var2.dim; - pass = pass && var1.type == var2.type; - pass = pass && var1.data.length() == var2.data.length(); - if ( pass ) { - for (size_t m=0; mvars[v], variable ); if ( pass ) { ut.passes(format+": Variable \"" + variable->name + "\" matched"); } else { diff --git a/tests/convertIO.cpp b/tests/convertIO.cpp index a6745263..4eb726b0 100644 --- a/tests/convertIO.cpp +++ b/tests/convertIO.cpp @@ -38,7 +38,7 @@ int main(int argc, char **argv) std::string path = IO::getPath( filename ); // Read the timesteps - auto timesteps = IO::readTimesteps( filename ); + auto timesteps = IO::readTimesteps( filename, "old" ); // Loop through the timesteps, reading/writing the data IO::initialize( "", format, false ); From de743e75b72c01fd3538976de81cc0f93e453974 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Wed, 17 Mar 2021 05:15:17 -0400 Subject: [PATCH 14/32] include convert IO --- tests/CMakeLists.txt | 4 ++-- tests/convertIO.cpp | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8df4e6bd..2c4b8431 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -35,8 +35,8 @@ ADD_LBPM_EXECUTABLE( GenerateSphereTest ) #ADD_LBPM_EXECUTABLE( BlobAnalysis ) #ADD_LBPM_EXECUTABLE( BlobIdentify ) #ADD_LBPM_EXECUTABLE( BlobIdentifyParallel ) -#ADD_LBPM_EXECUTABLE( convertIO ) -#ADD_LBPM_EXECUTABLE( DataAggregator ) +ADD_LBPM_EXECUTABLE( convertIO ) +ADD_LBPM_EXECUTABLE( DataAggregator ) #ADD_LBPM_EXECUTABLE( BlobAnalyzeParallel )( ADD_LBPM_EXECUTABLE( lbpm_minkowski_scalar ) ADD_LBPM_EXECUTABLE( TestPoissonSolver ) diff --git a/tests/convertIO.cpp b/tests/convertIO.cpp index 4eb726b0..c53ffe4f 100644 --- a/tests/convertIO.cpp +++ b/tests/convertIO.cpp @@ -5,7 +5,7 @@ #include #include -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "common/Communication.h" #include "common/Utilities.h" #include "IO/Mesh.h" @@ -13,7 +13,6 @@ #include "IO/Writer.h" #include "ProfilerApp.h" - int main(int argc, char **argv) { // Initialize MPI @@ -38,7 +37,7 @@ int main(int argc, char **argv) std::string path = IO::getPath( filename ); // Read the timesteps - auto timesteps = IO::readTimesteps( filename, "old" ); + auto timesteps = IO::readTimesteps( filename, "silo" ); // Loop through the timesteps, reading/writing the data IO::initialize( "", format, false ); @@ -63,19 +62,20 @@ int main(int argc, char **argv) // Read the variables for ( auto var : database.variables ) { auto varData = IO::getVariable( path, timestep, database, rank, var.name ); + printf("reformat %s \n",var.name); IO::reformatVariable( *meshData[i].mesh, *varData ); meshData[i].vars.push_back( varData ); } i++; } - MPI_Barrier(comm); + comm.barrier(); PROFILE_STOP("Read"); // Save the mesh data to a new file PROFILE_START("Write"); IO::writeData( timestep, meshData, MPI_COMM_WORLD ); - MPI_Barrier(comm); + comm.barrier(); PROFILE_STOP("Write"); } From cb32a4236bc00e1c7cecfe25a6a52ef39821eb76 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 17 Mar 2021 06:26:15 -0400 Subject: [PATCH 15/32] update lee model tests --- tests/TestMixedGrad.cpp | 89 ++++++++++++++++++++++++++++++++- tests/testGlobalMassFreeLee.cpp | 2 + 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/tests/TestMixedGrad.cpp b/tests/TestMixedGrad.cpp index bbf686ea..61fa6443 100644 --- a/tests/TestMixedGrad.cpp +++ b/tests/TestMixedGrad.cpp @@ -29,6 +29,7 @@ inline void Initialize_Mask(ScaLBL_FreeLeeModel &LeeModel){ } } + inline void Initialize_DummyPhaseField(ScaLBL_FreeLeeModel &LeeModel, double ax, double ay, double az){ // initialize a bubble int i,j,k,n; @@ -57,12 +58,94 @@ inline void Initialize_DummyPhaseField(ScaLBL_FreeLeeModel &LeeModel, double ax, LeeModel.MGTest(); } +inline int MultiHaloNeighborCheck(ScaLBL_FreeLeeModel &LeeModel){ + int i,j,k,iq,stride,nread; + int Nxh = LeeModel.Nxh; + int Nyh = LeeModel.Nyh; + int Np = LeeModel.Np; + + int *TmpMap; + TmpMap = new int[Np]; + ScaLBL_CopyToHost(TmpMap, LeeModel.dvcMap, Np*sizeof(int)); + + int *neighborList; + neighborList = new int[18*Np]; + ScaLBL_CopyToHost(neighborList, LeeModel.NeighborList, 18*Np*sizeof(int)); + printf("Check stride for interior neighbors \n"); + int count = 0; + for (int n=LeeModel.ScaLBL_Comm->LastInterior(); nLastInterior(); n++){ + // q=0 + int idx = TmpMap[n]; + k = idx/Nxh/Nyh; + j = (idx-k*Nxh*Nyh)/Nxh; + i = (idx-k*Nxh*Nyh -j*Nxh); + + // q=1 + nread = neighborList[n]; + iq = TmpMap[nread]; + stride = idx - iq; + if (stride != 1){ + printf(" %i, %i, %i q = 1 stride=%i \n ",i,j,k,stride); + count++; + } + + // q=2 + nread = neighborList[n+Np]; + iq = TmpMap[nread]; + stride = iq - idx; + if (stride != 1){ + printf(" %i, %i, %i q = 2 stride=%i \n ",i,j,k,stride); + count++; + } + + + // q=3 + nread = neighborList[n+2*Np]; + iq = TmpMap[nread]; + stride = idx - iq; + if (stride != Nxh){ + printf(" %i, %i, %i q = 3 stride=%i \n ",i,j,k,stride); + count++; + } + + // q = 4 + nread = neighborList[n+3*Np]; + iq = TmpMap[nread]; + stride = iq-idx; + if (stride != Nxh){ + printf(" %i, %i, %i q = 4 stride=%i \n ",i,j,k,stride); + count++; + } + + + // q=5 + nread = neighborList[n+4*Np]; + iq = TmpMap[nread]; + stride = idx - iq; + if (stride != Nxh*Nyh){ + count++; + printf(" %i, %i, %i q = 5 stride=%i \n ",i,j,k,stride); + } + + // q = 6 + nread = neighborList[n+5*Np]; + iq = TmpMap[nread]; + stride = iq - idx; + if (stride != Nxh*Nyh){ + count++; + printf(" %i, %i, %i q = 6 stride=%i \n ",i,j,k,stride); + } + + } + return count; +} + int main( int argc, char **argv ) { // Initialize Utilities::startup( argc, argv ); - + int errors = 0; // Load the input database auto db = std::make_shared( argv[1] ); @@ -98,6 +181,8 @@ int main( int argc, char **argv ) //LeeModel.Create_DummyPhase_MGTest(); LeeModel.Create_TwoFluid(); + errors=MultiHaloNeighborCheck(LeeModel); + Initialize_DummyPhaseField(LeeModel,1.0, 2.0, 3.0); LeeModel.WriteDebug_TwoFluid(); @@ -109,6 +194,6 @@ int main( int argc, char **argv ) } // Limit scope so variables that contain communicators will free before MPI_Finialize Utilities::shutdown(); - return 0; + return errors; } diff --git a/tests/testGlobalMassFreeLee.cpp b/tests/testGlobalMassFreeLee.cpp index bd54f106..c9073b9a 100644 --- a/tests/testGlobalMassFreeLee.cpp +++ b/tests/testGlobalMassFreeLee.cpp @@ -53,6 +53,8 @@ int main( int argc, char **argv ) LeeModel.ReadInput(); LeeModel.Create_TwoFluid(); LeeModel.Initialize_TwoFluid(); + /* check neighbors */ + /* Copy the initial density to test that global mass is conserved */ int Nx = LeeModel.Dm->Nx; From 8c026a2f6d7f6aaeec4fcb7de54140ea197dcc49 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 17 Mar 2021 06:35:19 -0400 Subject: [PATCH 16/32] test neighborlist --- tests/TestMixedGrad.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/TestMixedGrad.cpp b/tests/TestMixedGrad.cpp index 61fa6443..6baede7c 100644 --- a/tests/TestMixedGrad.cpp +++ b/tests/TestMixedGrad.cpp @@ -73,7 +73,7 @@ inline int MultiHaloNeighborCheck(ScaLBL_FreeLeeModel &LeeModel){ ScaLBL_CopyToHost(neighborList, LeeModel.NeighborList, 18*Np*sizeof(int)); printf("Check stride for interior neighbors \n"); int count = 0; - for (int n=LeeModel.ScaLBL_Comm->LastInterior(); nLastInterior(); n++){ + for (int n=LeeModel.ScaLBL_Comm->FirstInterior(); nLastInterior(); n++){ // q=0 int idx = TmpMap[n]; k = idx/Nxh/Nyh; @@ -82,7 +82,7 @@ inline int MultiHaloNeighborCheck(ScaLBL_FreeLeeModel &LeeModel){ // q=1 nread = neighborList[n]; - iq = TmpMap[nread]; + iq = TmpMap[nread%Np]; stride = idx - iq; if (stride != 1){ printf(" %i, %i, %i q = 1 stride=%i \n ",i,j,k,stride); @@ -91,7 +91,7 @@ inline int MultiHaloNeighborCheck(ScaLBL_FreeLeeModel &LeeModel){ // q=2 nread = neighborList[n+Np]; - iq = TmpMap[nread]; + iq = TmpMap[nread%Np]; stride = iq - idx; if (stride != 1){ printf(" %i, %i, %i q = 2 stride=%i \n ",i,j,k,stride); @@ -101,7 +101,7 @@ inline int MultiHaloNeighborCheck(ScaLBL_FreeLeeModel &LeeModel){ // q=3 nread = neighborList[n+2*Np]; - iq = TmpMap[nread]; + iq = TmpMap[nread%Np]; stride = idx - iq; if (stride != Nxh){ printf(" %i, %i, %i q = 3 stride=%i \n ",i,j,k,stride); @@ -110,7 +110,7 @@ inline int MultiHaloNeighborCheck(ScaLBL_FreeLeeModel &LeeModel){ // q = 4 nread = neighborList[n+3*Np]; - iq = TmpMap[nread]; + iq = TmpMap[nread%Np]; stride = iq-idx; if (stride != Nxh){ printf(" %i, %i, %i q = 4 stride=%i \n ",i,j,k,stride); @@ -120,7 +120,7 @@ inline int MultiHaloNeighborCheck(ScaLBL_FreeLeeModel &LeeModel){ // q=5 nread = neighborList[n+4*Np]; - iq = TmpMap[nread]; + iq = TmpMap[nread%Np]; stride = idx - iq; if (stride != Nxh*Nyh){ count++; @@ -129,7 +129,7 @@ inline int MultiHaloNeighborCheck(ScaLBL_FreeLeeModel &LeeModel){ // q = 6 nread = neighborList[n+5*Np]; - iq = TmpMap[nread]; + iq = TmpMap[nread%Np]; stride = iq - idx; if (stride != Nxh*Nyh){ count++; From 89704cbb107bc060a762ef854b09bb420eb2044d Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Wed, 17 Mar 2021 10:24:14 -0400 Subject: [PATCH 17/32] Updating convertIO --- IO/IOHelpers.h | 32 ++- IO/Mesh.cpp | 569 ++++++++++++++++++++++--------------------- IO/Mesh.h | 116 +++++---- IO/MeshDatabase.cpp | 511 +++++++++++++++++++------------------- IO/MeshDatabase.h | 68 +++--- IO/PIO.cpp | 131 +++++----- IO/PIO.h | 47 ++-- IO/PIO.hpp | 12 +- IO/PackData.cpp | 77 +++--- IO/PackData.h | 35 ++- IO/PackData.hpp | 150 ++++++------ IO/Reader.cpp | 456 ++++++++++++++++++---------------- IO/Reader.h | 50 ++-- IO/Writer.cpp | 484 ++++++++++++++++++++---------------- IO/Writer.h | 25 +- IO/netcdf.cpp | 385 +++++++++++++++-------------- IO/netcdf.h | 55 +++-- IO/silo.cpp | 81 +++--- IO/silo.h | 121 ++++----- tests/CMakeLists.txt | 2 +- tests/TestWriter.cpp | 2 +- tests/convertIO.cpp | 113 ++++----- 22 files changed, 1843 insertions(+), 1679 deletions(-) diff --git a/IO/IOHelpers.h b/IO/IOHelpers.h index 2e9b06e0..4389c619 100644 --- a/IO/IOHelpers.h +++ b/IO/IOHelpers.h @@ -10,9 +10,9 @@ namespace IO { // Find a character in a line inline size_t find( const char *line, char token ) { - size_t i=0; + size_t i = 0; while ( 1 ) { - if ( line[i]==token || line[i]<32 || line[i]==0 ) + if ( line[i] == token || line[i] < 32 || line[i] == 0 ) break; ++i; } @@ -21,17 +21,17 @@ inline size_t find( const char *line, char token ) // Remove preceeding/trailing whitespace -inline std::string deblank( const std::string& str ) +inline std::string deblank( const std::string &str ) { size_t i1 = str.size(); size_t i2 = 0; - for (size_t i=0; i=32 ) { - i1 = std::min(i1,i); - i2 = std::max(i2,i); + for ( size_t i = 0; i < str.size(); i++ ) { + if ( str[i] != ' ' && str[i] >= 32 ) { + i1 = std::min( i1, i ); + i2 = std::max( i2, i ); } } - return str.substr(i1,i2-i1+1); + return str.substr( i1, i2 - i1 + 1 ); } @@ -42,14 +42,14 @@ inline std::vector splitList( const char *line, const char token ) size_t i1 = 0; size_t i2 = 0; while ( 1 ) { - if ( line[i2]==token || line[i2]<32 ) { - std::string tmp(&line[i1],i2-i1); - tmp = deblank(tmp); + if ( line[i2] == token || line[i2] < 32 ) { + std::string tmp( &line[i1], i2 - i1 ); + tmp = deblank( tmp ); if ( !tmp.empty() ) - list.push_back(tmp); - i1 = i2+1; + list.push_back( tmp ); + i1 = i2 + 1; } - if ( line[i2]==0 ) + if ( line[i2] == 0 ) break; i2++; } @@ -57,8 +57,6 @@ inline std::vector splitList( const char *line, const char token ) } - -}; +}; // namespace IO #endif - diff --git a/IO/Mesh.cpp b/IO/Mesh.cpp index eb712296..91c78e03 100644 --- a/IO/Mesh.cpp +++ b/IO/Mesh.cpp @@ -19,104 +19,98 @@ inline Point nullPoint() /**************************************************** -* Mesh * -****************************************************/ -Mesh::Mesh( ) -{ -} -Mesh::~Mesh( ) -{ -} + * Mesh * + ****************************************************/ +Mesh::Mesh() {} +Mesh::~Mesh() {} /**************************************************** -* MeshDataStruct * -****************************************************/ + * MeshDataStruct * + ****************************************************/ bool MeshDataStruct::check() const { - enum VariableType { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 }; bool pass = mesh != nullptr; - for ( const auto& var : vars ) { - pass = pass && static_cast(var->type)>=1 && static_cast(var->type)<=3; + for ( const auto &var : vars ) { + pass = pass && static_cast( var->type ) >= 1 && static_cast( var->type ) <= 3; pass = pass && !var->data.empty(); } - if ( !pass ) + if ( !pass ) { + std::cerr << "Invalid variable detected\n"; return false; - const std::string& meshClass = mesh->className(); + } + const std::string &meshClass = mesh->className(); if ( meshClass == "PointList" ) { - const auto mesh2 = dynamic_cast( mesh.get() ); - if ( mesh2 == nullptr ) - return false; - for ( const auto& var : vars ) { + auto mesh2 = dynamic_cast( mesh.get() ); + ASSERT( mesh2 ); + for ( const auto &var : vars ) { if ( var->type == IO::VariableType::NodeVariable ) { - pass = pass && var->data.size(0)==mesh2->points.size() && var->data.size(1)==var->dim; + pass = pass && var->data.size() == ArraySize( mesh2->points.size(), var->dim ); } else if ( var->type == IO::VariableType::EdgeVariable ) { - ERROR("Invalid type for PointList"); + ERROR( "Invalid type for PointList" ); } else if ( var->type == IO::VariableType::SurfaceVariable ) { - ERROR("Invalid type for PointList"); + ERROR( "Invalid type for PointList" ); } else if ( var->type == IO::VariableType::VolumeVariable ) { - ERROR("Invalid type for PointList"); + ERROR( "Invalid type for PointList" ); } else { - ERROR("Invalid variable type"); + ERROR( "Invalid variable type" ); } } } else if ( meshClass == "TriMesh" || meshClass == "TriList" ) { - const auto mesh2 = getTriMesh( mesh ); - if ( mesh2 == nullptr ) - return false; - for ( const auto& var : vars ) { + auto mesh2 = getTriMesh( mesh ); + ASSERT( mesh2 ); + for ( const auto &var : vars ) { if ( var->type == IO::VariableType::NodeVariable ) { - pass = pass && var->data.size(0)==mesh2->vertices->points.size() && var->data.size(1)==var->dim; + pass = pass && + var->data.size() == ArraySize( mesh2->vertices->points.size(), var->dim ); } else if ( var->type == IO::VariableType::EdgeVariable ) { - ERROR("Not finished"); + ERROR( "Not finished" ); } else if ( var->type == IO::VariableType::SurfaceVariable ) { - ERROR("Not finished"); + ERROR( "Not finished" ); } else if ( var->type == IO::VariableType::VolumeVariable ) { - pass = pass && var->data.size(0)==mesh2->A.size() && var->data.size(1)==var->dim; + pass = pass && var->data.size( 0 ) == mesh2->A.size() && + var->data.size( 1 ) == var->dim; } else { - ERROR("Invalid variable type"); + ERROR( "Invalid variable type" ); } } } else if ( meshClass == "DomainMesh" ) { - const auto mesh2 = dynamic_cast( mesh.get() ); - if ( mesh2 == nullptr ) - return false; - for ( const auto& var : vars ) { + auto mesh2 = dynamic_cast( mesh.get() ); + ASSERT( mesh2 ); + for ( const auto &var : vars ) { + ArraySize varSize; if ( var->type == IO::VariableType::NodeVariable ) { - pass = pass && (int) var->data.size(0)==(mesh2->nx+1) && (int) var->data.size(1)==(mesh2->ny+1) - && (int) var->data.size(2)==(mesh2->nz+1) && var->data.size(3)==var->dim; + varSize = ArraySize( mesh2->nx + 1, mesh2->ny + 1, mesh2->nz + 1, var->dim ); } else if ( var->type == IO::VariableType::EdgeVariable ) { - ERROR("Not finished"); + ERROR( "Not finished" ); } else if ( var->type == IO::VariableType::SurfaceVariable ) { - ERROR("Not finished"); + ERROR( "Not finished" ); } else if ( var->type == IO::VariableType::VolumeVariable ) { - pass = pass && (int) var->data.size(0)==mesh2->nx && (int) var->data.size(1)==mesh2->ny - && (int) var->data.size(2)==mesh2->nz && var->data.size(3)==var->dim; + varSize = ArraySize( mesh2->nx, mesh2->ny, mesh2->nz, var->dim ); } else { - ERROR("Invalid variable type"); + ERROR( "Invalid variable type" ); } + if ( var->data.size() == ArraySize( varSize[0] * varSize[1] * varSize[2], varSize[3] ) ) + var->data.resize( varSize ); + pass = pass && var->data.size() == varSize; } } else { - ERROR("Unknown mesh class: "+mesh->className()); + ERROR( "Unknown mesh class: " + mesh->className() ); } return pass; } /**************************************************** -* PointList * -****************************************************/ -PointList::PointList( ) -{ -} + * PointList * + ****************************************************/ +PointList::PointList() {} PointList::PointList( size_t N ) { Point tmp = nullPoint(); - points.resize(N,tmp); -} -PointList::~PointList( ) -{ + points.resize( N, tmp ); } +PointList::~PointList() {} size_t PointList::numberPointsVar( VariableType type ) const { size_t N = 0; @@ -124,174 +118,168 @@ size_t PointList::numberPointsVar( VariableType type ) const N = points.size(); return N; } -std::pair PointList::pack( int level ) const +std::pair PointList::pack( int level ) const { - std::pair data_out(0,NULL); - if ( level==0 ) { - data_out.first = (2+3*points.size())*sizeof(double); - double *data_ptr = new double[2+3*points.size()]; - data_out.second = data_ptr; - uint64_t *data_int = reinterpret_cast(data_ptr); - data_int[0] = level; - data_int[1] = points.size(); - double *data = &data_ptr[2]; - for (size_t i=0; i data_out( 0, NULL ); + if ( level == 0 ) { + data_out.first = ( 2 + 3 * points.size() ) * sizeof( double ); + double *data_ptr = new double[2 + 3 * points.size()]; + data_out.second = data_ptr; + uint64_t *data_int = reinterpret_cast( data_ptr ); + data_int[0] = level; + data_int[1] = points.size(); + double *data = &data_ptr[2]; + for ( size_t i = 0; i < points.size(); i++ ) { + data[3 * i + 0] = points[i].x; + data[3 * i + 1] = points[i].y; + data[3 * i + 2] = points[i].z; } } return data_out; } -void PointList::unpack( const std::pair& data_in ) +void PointList::unpack( const std::pair &data_in ) { - uint64_t *data_int = reinterpret_cast(data_in.second); - const double *data = reinterpret_cast(data_in.second); - int level = data_int[0]; - uint64_t N = data_int[1]; - data = &data[2]; - if ( level==0 ) { - ASSERT((2+3*N)*sizeof(double)==data_in.first); - points.resize(N); - for (size_t i=0; i( data_in.second ); + const double *data = reinterpret_cast( data_in.second ); + int level = data_int[0]; + uint64_t N = data_int[1]; + data = &data[2]; + if ( level == 0 ) { + ASSERT( ( 2 + 3 * N ) * sizeof( double ) == data_in.first ); + points.resize( N ); + for ( size_t i = 0; i < points.size(); i++ ) { + points[i].x = data[3 * i + 0]; + points[i].y = data[3 * i + 1]; + points[i].z = data[3 * i + 2]; } } } /**************************************************** -* TriList * -****************************************************/ -TriList::TriList( ) -{ -} + * TriList * + ****************************************************/ +TriList::TriList() {} TriList::TriList( size_t N_tri ) { Point tmp = nullPoint(); - A.resize(N_tri,tmp); - B.resize(N_tri,tmp); - C.resize(N_tri,tmp); + A.resize( N_tri, tmp ); + B.resize( N_tri, tmp ); + C.resize( N_tri, tmp ); } -TriList::TriList( const TriMesh& mesh ) +TriList::TriList( const TriMesh &mesh ) { Point tmp = nullPoint(); - A.resize(mesh.A.size(),tmp); - B.resize(mesh.B.size(),tmp); - C.resize(mesh.C.size(),tmp); - ASSERT(mesh.vertices.get()!=NULL); - const std::vector& P = mesh.vertices->points; - for (size_t i=0; i &P = mesh.vertices->points; + for ( size_t i = 0; i < A.size(); i++ ) A[i] = P[mesh.A[i]]; - for (size_t i=0; i TriList::pack( int level ) const +std::pair TriList::pack( int level ) const { - std::pair data_out(0,NULL); - if ( level==0 ) { - data_out.first = (2+9*A.size())*sizeof(double); - double *data_ptr = new double[2+9*A.size()]; - data_out.second = data_ptr; - uint64_t *data_int = reinterpret_cast(data_ptr); - data_int[0] = level; - data_int[1] = A.size(); - double *data = &data_ptr[2]; - for (size_t i=0; i data_out( 0, NULL ); + if ( level == 0 ) { + data_out.first = ( 2 + 9 * A.size() ) * sizeof( double ); + double *data_ptr = new double[2 + 9 * A.size()]; + data_out.second = data_ptr; + uint64_t *data_int = reinterpret_cast( data_ptr ); + data_int[0] = level; + data_int[1] = A.size(); + double *data = &data_ptr[2]; + for ( size_t i = 0; i < A.size(); i++ ) { + data[9 * i + 0] = A[i].x; + data[9 * i + 1] = A[i].y; + data[9 * i + 2] = A[i].z; + data[9 * i + 3] = B[i].x; + data[9 * i + 4] = B[i].y; + data[9 * i + 5] = B[i].z; + data[9 * i + 6] = C[i].x; + data[9 * i + 7] = C[i].y; + data[9 * i + 8] = C[i].z; } } return data_out; } -void TriList::unpack( const std::pair& data_in ) +void TriList::unpack( const std::pair &data_in ) { - uint64_t *data_int = reinterpret_cast(data_in.second); - const double *data = reinterpret_cast(data_in.second); - int level = data_int[0]; - uint64_t N = data_int[1]; - data = &data[2]; - if ( level==0 ) { - ASSERT((2+9*N)*sizeof(double)==data_in.first); - A.resize(N); - B.resize(N); - C.resize(N); - for (size_t i=0; i( data_in.second ); + const double *data = reinterpret_cast( data_in.second ); + int level = data_int[0]; + uint64_t N = data_int[1]; + data = &data[2]; + if ( level == 0 ) { + ASSERT( ( 2 + 9 * N ) * sizeof( double ) == data_in.first ); + A.resize( N ); + B.resize( N ); + C.resize( N ); + for ( size_t i = 0; i < A.size(); i++ ) { + A[i].x = data[9 * i + 0]; + A[i].y = data[9 * i + 1]; + A[i].z = data[9 * i + 2]; + B[i].x = data[9 * i + 3]; + B[i].y = data[9 * i + 4]; + B[i].z = data[9 * i + 5]; + C[i].x = data[9 * i + 6]; + C[i].y = data[9 * i + 7]; + C[i].z = data[9 * i + 8]; } } } /**************************************************** -* TriMesh * -****************************************************/ -TriMesh::TriMesh( ) -{ -} + * TriMesh * + ****************************************************/ +TriMesh::TriMesh() {} TriMesh::TriMesh( size_t N_tri, size_t N_point ) { - vertices.reset( new PointList(N_point) ); - A.resize(N_tri,-1); - B.resize(N_tri,-1); - C.resize(N_tri,-1); + vertices.reset( new PointList( N_point ) ); + A.resize( N_tri, -1 ); + B.resize( N_tri, -1 ); + C.resize( N_tri, -1 ); } TriMesh::TriMesh( size_t N_tri, std::shared_ptr points ) { vertices = points; - A.resize(N_tri,-1); - B.resize(N_tri,-1); - C.resize(N_tri,-1); + A.resize( N_tri, -1 ); + B.resize( N_tri, -1 ); + C.resize( N_tri, -1 ); } -TriMesh::TriMesh( const TriList& mesh ) +TriMesh::TriMesh( const TriList &mesh ) { // For simlicity we will just create a mesh with ~3x the verticies for now - ASSERT(mesh.A.size()==mesh.B.size()&&mesh.A.size()==mesh.C.size()); - A.resize(mesh.A.size()); - B.resize(mesh.B.size()); - C.resize(mesh.C.size()); - vertices.reset( new PointList(3*mesh.A.size()) ); - for (size_t i=0; ipoints[A[i]] = mesh.A[i]; vertices->points[B[i]] = mesh.B[i]; vertices->points[C[i]] = mesh.C[i]; } } -TriMesh::~TriMesh( ) +TriMesh::~TriMesh() { vertices.reset(); A.clear(); @@ -301,181 +289,194 @@ TriMesh::~TriMesh( ) size_t TriMesh::numberPointsVar( VariableType type ) const { size_t N = 0; - if ( type==VariableType::NodeVariable ) + if ( type == VariableType::NodeVariable ) N = vertices->points.size(); - else if ( type==VariableType::SurfaceVariable || type==VariableType::VolumeVariable ) + else if ( type == VariableType::SurfaceVariable || type == VariableType::VolumeVariable ) N = A.size(); return N; } -std::pair TriMesh::pack( int level ) const +std::pair TriMesh::pack( int level ) const { - std::pair data_out(0,NULL); - if ( level==0 ) { - const std::vector& points = vertices->points; - data_out.first = (3+3*points.size())*sizeof(double) + 3*A.size()*sizeof(int); - double *data_ptr = new double[4+3*points.size()+(3*A.size()*sizeof(int))/sizeof(double)]; - data_out.second = data_ptr; - uint64_t *data_int64 = reinterpret_cast(data_ptr); - data_int64[0] = level; - data_int64[1] = points.size(); - data_int64[2] = A.size(); - double *data = &data_ptr[3]; - for (size_t i=0; i data_out( 0, NULL ); + if ( level == 0 ) { + const std::vector &points = vertices->points; + data_out.first = + ( 3 + 3 * points.size() ) * sizeof( double ) + 3 * A.size() * sizeof( int ); + double *data_ptr = + new double[4 + 3 * points.size() + ( 3 * A.size() * sizeof( int ) ) / sizeof( double )]; + data_out.second = data_ptr; + uint64_t *data_int64 = reinterpret_cast( data_ptr ); + data_int64[0] = level; + data_int64[1] = points.size(); + data_int64[2] = A.size(); + double *data = &data_ptr[3]; + for ( size_t i = 0; i < points.size(); i++ ) { + data[3 * i + 0] = points[i].x; + data[3 * i + 1] = points[i].y; + data[3 * i + 2] = points[i].z; } - int *data_int = reinterpret_cast(&data[3*points.size()]); - for (size_t i=0; i( &data[3 * points.size()] ); + for ( size_t i = 0; i < A.size(); i++ ) { + data_int[3 * i + 0] = A[i]; + data_int[3 * i + 1] = B[i]; + data_int[3 * i + 2] = C[i]; } } return data_out; } -void TriMesh::unpack( const std::pair& data_in ) +void TriMesh::unpack( const std::pair &data_in ) { - uint64_t *data_int64 = reinterpret_cast(data_in.second); - const double *data = reinterpret_cast(data_in.second); - int level = data_int64[0]; - uint64_t N_P = data_int64[1]; - uint64_t N_A = data_int64[2]; - data = &data[3]; - if ( level==0 ) { - size_t size = (3+3*N_P)*sizeof(double)+3*N_A*sizeof(int); - ASSERT(size==data_in.first); - vertices.reset( new PointList(N_P) ); - std::vector& points = vertices->points; - for (size_t i=0; i( data_in.second ); + const double *data = reinterpret_cast( data_in.second ); + int level = data_int64[0]; + uint64_t N_P = data_int64[1]; + uint64_t N_A = data_int64[2]; + data = &data[3]; + if ( level == 0 ) { + size_t size = ( 3 + 3 * N_P ) * sizeof( double ) + 3 * N_A * sizeof( int ); + ASSERT( size == data_in.first ); + vertices.reset( new PointList( N_P ) ); + std::vector &points = vertices->points; + for ( size_t i = 0; i < points.size(); i++ ) { + points[i].x = data[3 * i + 0]; + points[i].y = data[3 * i + 1]; + points[i].z = data[3 * i + 2]; } - const int *data_int = reinterpret_cast(&data[3*N_P]); - A.resize(N_A); - B.resize(N_A); - C.resize(N_A); - for (size_t i=0; i( &data[3 * N_P] ); + A.resize( N_A ); + B.resize( N_A ); + C.resize( N_A ); + for ( size_t i = 0; i < A.size(); i++ ) { + A[i] = data_int[3 * i + 0]; + B[i] = data_int[3 * i + 1]; + C[i] = data_int[3 * i + 2]; } } } /**************************************************** -* Domain mesh * -****************************************************/ -DomainMesh::DomainMesh(): - nprocx(0), nprocy(0), nprocz(0), rank(0), - nx(0), ny(0), nz(0), - Lx(0), Ly(0), Lz(0) + * Domain mesh * + ****************************************************/ +DomainMesh::DomainMesh() + : nprocx( 0 ), + nprocy( 0 ), + nprocz( 0 ), + rank( 0 ), + nx( 0 ), + ny( 0 ), + nz( 0 ), + Lx( 0 ), + Ly( 0 ), + Lz( 0 ) { } -DomainMesh::DomainMesh( RankInfoStruct data, - int nx2, int ny2, int nz2, double Lx2, double Ly2, double Lz2 ): - nprocx(data.nx), nprocy(data.ny), nprocz(data.nz), rank(data.rank[1][1][1]), - nx(nx2), ny(ny2), nz(nz2), - Lx(Lx2), Ly(Ly2), Lz(Lz2) -{ -} -DomainMesh::~DomainMesh() +DomainMesh::DomainMesh( + RankInfoStruct data, int nx2, int ny2, int nz2, double Lx2, double Ly2, double Lz2 ) + : nprocx( data.nx ), + nprocy( data.ny ), + nprocz( data.nz ), + rank( data.rank[1][1][1] ), + nx( nx2 ), + ny( ny2 ), + nz( nz2 ), + Lx( Lx2 ), + Ly( Ly2 ), + Lz( Lz2 ) { } +DomainMesh::~DomainMesh() {} size_t DomainMesh::numberPointsVar( VariableType type ) const { size_t N = 0; - if ( type==VariableType::NodeVariable ) - N = (nx+1)*(ny+1)*(nz+1); - else if ( type==VariableType::SurfaceVariable ) - N = (nx+1)*ny*nz + nx*(ny+1)*nz + nx*ny*(nz+1); - else if ( type==VariableType::VolumeVariable ) - N = nx*ny*nz; + if ( type == VariableType::NodeVariable ) + N = ( nx + 1 ) * ( ny + 1 ) * ( nz + 1 ); + else if ( type == VariableType::SurfaceVariable ) + N = ( nx + 1 ) * ny * nz + nx * ( ny + 1 ) * nz + nx * ny * ( nz + 1 ); + else if ( type == VariableType::VolumeVariable ) + N = nx * ny * nz; return N; } -std::pair DomainMesh::pack( int level ) const +std::pair DomainMesh::pack( int level ) const { - std::pair data(0,NULL); - data.first = 7*sizeof(double); + std::pair data( 0, NULL ); + data.first = 7 * sizeof( double ); data.second = new double[7]; - memset(data.second,0,7*sizeof(double)); - int *data_int = reinterpret_cast(data.second); - double *data_double = &reinterpret_cast(data.second)[4]; - data_int[0] = nprocx; - data_int[1] = nprocy; - data_int[2] = nprocz; - data_int[3] = rank; - data_int[4] = nx; - data_int[5] = ny; - data_int[6] = nz; - data_double[0] = Lx; - data_double[1] = Ly; - data_double[2] = Lz; + memset( data.second, 0, 7 * sizeof( double ) ); + int *data_int = reinterpret_cast( data.second ); + double *data_double = &reinterpret_cast( data.second )[4]; + data_int[0] = nprocx; + data_int[1] = nprocy; + data_int[2] = nprocz; + data_int[3] = rank; + data_int[4] = nx; + data_int[5] = ny; + data_int[6] = nz; + data_double[0] = Lx; + data_double[1] = Ly; + data_double[2] = Lz; return data; } -void DomainMesh::unpack( const std::pair& data ) +void DomainMesh::unpack( const std::pair &data ) { - const int *data_int = reinterpret_cast(data.second); - const double *data_double = &reinterpret_cast(data.second)[4]; - nprocx = data_int[0]; - nprocy = data_int[1]; - nprocz = data_int[2]; - rank = data_int[3]; - nx = data_int[4]; - ny = data_int[5]; - nz = data_int[6]; - Lx = data_double[0]; - Ly = data_double[1]; - Lz = data_double[2]; + const int *data_int = reinterpret_cast( data.second ); + const double *data_double = &reinterpret_cast( data.second )[4]; + nprocx = data_int[0]; + nprocy = data_int[1]; + nprocz = data_int[2]; + rank = data_int[3]; + nx = data_int[4]; + ny = data_int[5]; + nz = data_int[6]; + Lx = data_double[0]; + Ly = data_double[1]; + Lz = data_double[2]; } /**************************************************** -* Converters * -****************************************************/ + * Converters * + ****************************************************/ std::shared_ptr getPointList( std::shared_ptr mesh ) { - return std::dynamic_pointer_cast(mesh); + return std::dynamic_pointer_cast( mesh ); } std::shared_ptr getTriMesh( std::shared_ptr mesh ) { std::shared_ptr mesh2; - if ( std::dynamic_pointer_cast(mesh).get() != NULL ) { - mesh2 = std::dynamic_pointer_cast(mesh); - } else if ( std::dynamic_pointer_cast(mesh).get() != NULL ) { - std::shared_ptr trilist = std::dynamic_pointer_cast(mesh); - ASSERT(trilist.get()!=NULL); - mesh2.reset( new TriMesh(*trilist) ); + if ( std::dynamic_pointer_cast( mesh ).get() != NULL ) { + mesh2 = std::dynamic_pointer_cast( mesh ); + } else if ( std::dynamic_pointer_cast( mesh ).get() != NULL ) { + std::shared_ptr trilist = std::dynamic_pointer_cast( mesh ); + ASSERT( trilist.get() != NULL ); + mesh2.reset( new TriMesh( *trilist ) ); } return mesh2; } std::shared_ptr getTriList( std::shared_ptr mesh ) { std::shared_ptr mesh2; - if ( std::dynamic_pointer_cast(mesh).get() != NULL ) { - mesh2 = std::dynamic_pointer_cast(mesh); - } else if ( std::dynamic_pointer_cast(mesh).get() != NULL ) { - std::shared_ptr trimesh = std::dynamic_pointer_cast(mesh); - ASSERT(trimesh.get()!=NULL); - mesh2.reset( new TriList(*trimesh) ); + if ( std::dynamic_pointer_cast( mesh ).get() != NULL ) { + mesh2 = std::dynamic_pointer_cast( mesh ); + } else if ( std::dynamic_pointer_cast( mesh ).get() != NULL ) { + std::shared_ptr trimesh = std::dynamic_pointer_cast( mesh ); + ASSERT( trimesh.get() != NULL ); + mesh2.reset( new TriList( *trimesh ) ); } return mesh2; } std::shared_ptr getPointList( std::shared_ptr mesh ) { - return getPointList( std::const_pointer_cast(mesh) ); + return getPointList( std::const_pointer_cast( mesh ) ); } std::shared_ptr getTriMesh( std::shared_ptr mesh ) { - return getTriMesh( std::const_pointer_cast(mesh) ); + return getTriMesh( std::const_pointer_cast( mesh ) ); } std::shared_ptr getTriList( std::shared_ptr mesh ) { - return getTriList( std::const_pointer_cast(mesh) ); + return getTriList( std::const_pointer_cast( mesh ) ); } -} // IO namespace - +} // namespace IO diff --git a/IO/Mesh.h b/IO/Mesh.h index b204675a..a60e14c9 100644 --- a/IO/Mesh.h +++ b/IO/Mesh.h @@ -6,17 +6,23 @@ #include #include +#include "analysis/PointList.h" #include "common/Array.h" #include "common/Communication.h" -#include "analysis/PointList.h" namespace IO { //! Possible variable types -enum class VariableType: unsigned char { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 }; -enum class DataType: unsigned char { Double=1, Float=2, Int=2, Null=0 }; +enum class VariableType : unsigned char { + NodeVariable = 1, + EdgeVariable = 2, + SurfaceVariable = 3, + VolumeVariable = 4, + NullVariable = 0 +}; +enum class DataType : unsigned char { Double = 1, Float = 2, Int = 2, Null = 0 }; /*! \class Mesh @@ -32,21 +38,22 @@ public: //! Number of points for the given variable type virtual size_t numberPointsVar( VariableType type ) const = 0; //! Pack the data - virtual std::pair pack( int level ) const = 0; + virtual std::pair pack( int level ) const = 0; //! Unpack the data - virtual void unpack( const std::pair& data ) = 0; + virtual void unpack( const std::pair &data ) = 0; + protected: //! Empty constructor Mesh(); - Mesh(const Mesh&); - Mesh& operator=(const Mesh&); + Mesh( const Mesh & ); + Mesh &operator=( const Mesh & ); }; /*! \class PointList \brief A class used to hold a list of verticies */ -class PointList: public Mesh +class PointList : public Mesh { public: //! Empty constructor @@ -60,13 +67,14 @@ public: //! Number of points for the given variable type virtual size_t numberPointsVar( VariableType type ) const; //! Pack the data - virtual std::pair pack( int level ) const; + virtual std::pair pack( int level ) const; //! Unpack the data - virtual void unpack( const std::pair& data ); + virtual void unpack( const std::pair &data ); //! Access the points - const std::vector& getPoints() const { return points; } + const std::vector &getPoints() const { return points; } + public: - std::vector points; //!< List of points vertex + std::vector points; //!< List of points vertex }; @@ -74,7 +82,7 @@ public: \brief A class used to hold a list of triangles specified by their vertex coordinates */ class TriMesh; -class TriList: public Mesh +class TriList : public Mesh { public: //! Empty constructor @@ -82,7 +90,7 @@ public: //! Constructor for N triangles TriList( size_t N_tri ); //! Constructor from TriMesh - TriList( const TriMesh& ); + TriList( const TriMesh & ); //! Destructor virtual ~TriList(); //! Mesh class name @@ -90,20 +98,22 @@ public: //! Number of points for the given variable type virtual size_t numberPointsVar( VariableType type ) const; //! Pack the data - virtual std::pair pack( int level ) const; + virtual std::pair pack( int level ) const; //! Unpack the data - virtual void unpack( const std::pair& data ); + virtual void unpack( const std::pair &data ); + public: - std::vector A; //!< First vertex - std::vector B; //!< Second vertex - std::vector C; //!< Third vertex + std::vector A; //!< First vertex + std::vector B; //!< Second vertex + std::vector C; //!< Third vertex }; /*! \class TriMesh - \brief A class used to hold a list of trianges specified by their vertex number and list of coordiantes + \brief A class used to hold a list of trianges specified by their vertex number and list of + coordiantes */ -class TriMesh: public Mesh +class TriMesh : public Mesh { public: //! TriMesh constructor @@ -113,7 +123,7 @@ public: //! Constructor for Nt triangles and the given points TriMesh( size_t N_tri, std::shared_ptr points ); //! Constructor from TriList - TriMesh( const TriList& ); + TriMesh( const TriList & ); //! Destructor virtual ~TriMesh(); //! Mesh class name @@ -121,21 +131,22 @@ public: //! Number of points for the given variable type virtual size_t numberPointsVar( VariableType type ) const; //! Pack the data - virtual std::pair pack( int level ) const; + virtual std::pair pack( int level ) const; //! Unpack the data - virtual void unpack( const std::pair& data ); + virtual void unpack( const std::pair &data ); + public: - std::shared_ptr vertices; //!< List of verticies - std::vector A; //!< First vertex - std::vector B; //!< Second vertex - std::vector C; //!< Third vertex + std::shared_ptr vertices; //!< List of verticies + std::vector A; //!< First vertex + std::vector B; //!< Second vertex + std::vector C; //!< Third vertex }; /*! \class Domain \brief A class used to hold the domain */ -class DomainMesh: public Mesh +class DomainMesh : public Mesh { public: //! Empty constructor @@ -149,9 +160,10 @@ public: //! Number of points for the given variable type virtual size_t numberPointsVar( VariableType type ) const; //! Pack the data - virtual std::pair pack( int level ) const; + virtual std::pair pack( int level ) const; //! Unpack the data - virtual void unpack( const std::pair& data ); + virtual void unpack( const std::pair &data ); + public: int nprocx, nprocy, nprocz, rank; int nx, ny, nz; @@ -159,37 +171,40 @@ public: }; - /*! \class Variable \brief A base class for variables */ -struct Variable -{ +struct Variable { public: // Internal variables - unsigned char dim; //!< Number of points per grid point (1: scalar, 3: vector, ...) - VariableType type; //!< Variable type - DataType precision; //!< Variable precision to use for IO - std::string name; //!< Variable name - Array data; //!< Variable data + unsigned char dim; //!< Number of points per grid point (1: scalar, 3: vector, ...) + VariableType type; //!< Variable type + DataType precision; //!< Variable precision to use for IO + std::string name; //!< Variable name + Array data; //!< Variable data //! Empty constructor - Variable(): dim(0), type(VariableType::NullVariable), precision(DataType::Double) {} + Variable() : dim( 0 ), type( VariableType::NullVariable ), precision( DataType::Double ) {} //! Constructor - Variable( int dim_, IO::VariableType type_, const std::string& name_ ): - dim(dim_), type(type_), precision(DataType::Double), name(name_) {} + Variable( int dim_, IO::VariableType type_, const std::string &name_ ) + : dim( dim_ ), type( type_ ), precision( DataType::Double ), name( name_ ) + { + } //! Constructor - Variable( int dim_, IO::VariableType type_, const std::string& name_, const Array& data_ ): - dim(dim_), type(type_), precision(DataType::Double), name(name_), data(data_) {} + Variable( + int dim_, IO::VariableType type_, const std::string &name_, const Array &data_ ) + : dim( dim_ ), type( type_ ), precision( DataType::Double ), name( name_ ), data( data_ ) + { + } //! Destructor virtual ~Variable() {} + protected: //! Empty constructor - Variable(const Variable&); - Variable& operator=(const Variable&); + Variable( const Variable & ); + Variable &operator=( const Variable & ); }; - /*! \class MeshDataStruct \brief A class used to hold database info for saving a mesh */ @@ -197,9 +212,9 @@ struct MeshDataStruct { DataType precision; //!< Precision to use for IO (mesh) std::string meshName; //!< Mesh name std::shared_ptr mesh; //!< Mesh data - std::vector > vars; + std::vector> vars; //! Empty constructor - MeshDataStruct(): precision(DataType::Double) {} + MeshDataStruct() : precision( DataType::Double ) {} //! Check the data bool check() const; }; @@ -214,7 +229,6 @@ std::shared_ptr getTriMesh( std::shared_ptr mesh ); std::shared_ptr getTriList( std::shared_ptr mesh ); -} // IO namespace +} // namespace IO #endif - diff --git a/IO/MeshDatabase.cpp b/IO/MeshDatabase.cpp index 2c03ddde..70b9acc3 100644 --- a/IO/MeshDatabase.cpp +++ b/IO/MeshDatabase.cpp @@ -1,130 +1,151 @@ #include "IO/MeshDatabase.h" +#include "IO/IOHelpers.h" #include "IO/Mesh.h" #include "IO/PackData.h" -#include "IO/IOHelpers.h" #include "common/MPI.h" #include "common/Utilities.h" -#include +#include #include #include -#include +#include #include - // MeshType template<> -size_t packsize( const IO::MeshType& rhs ) +size_t packsize( const IO::MeshType &rhs ) { - return sizeof(IO::MeshType); + return sizeof( IO::MeshType ); } template<> -void pack( const IO::MeshType& rhs, char *buffer ) +void pack( const IO::MeshType &rhs, char *buffer ) { - memcpy(buffer,&rhs,sizeof(IO::MeshType)); + memcpy( buffer, &rhs, sizeof( IO::MeshType ) ); } template<> -void unpack( IO::MeshType& data, const char *buffer ) +void unpack( IO::MeshType &data, const char *buffer ) { - memcpy(&data,buffer,sizeof(IO::MeshType)); + memcpy( &data, buffer, sizeof( IO::MeshType ) ); } // Variable::VariableType template<> -size_t packsize( const IO::VariableType& rhs ) +size_t packsize( const IO::VariableType &rhs ) { - return sizeof(IO::VariableType); + return sizeof( IO::VariableType ); } template<> -void pack( const IO::VariableType& rhs, char *buffer ) +void pack( const IO::VariableType &rhs, char *buffer ) { - memcpy(buffer,&rhs,sizeof(IO::VariableType)); + memcpy( buffer, &rhs, sizeof( IO::VariableType ) ); } template<> -void unpack( IO::VariableType& data, const char *buffer ) +void unpack( IO::VariableType &data, const char *buffer ) { - memcpy(&data,buffer,sizeof(IO::VariableType)); + memcpy( &data, buffer, sizeof( IO::VariableType ) ); } // DatabaseEntry template<> -size_t packsize( const IO::DatabaseEntry& rhs ) +size_t packsize( const IO::DatabaseEntry &rhs ) { - return packsize(rhs.name)+packsize(rhs.file)+packsize(rhs.offset); + return packsize( rhs.name ) + packsize( rhs.file ) + packsize( rhs.offset ); } template<> -void pack( const IO::DatabaseEntry& rhs, char *buffer ) +void pack( const IO::DatabaseEntry &rhs, char *buffer ) { - size_t i=0; - pack(rhs.name,&buffer[i]); i+=packsize(rhs.name); - pack(rhs.file,&buffer[i]); i+=packsize(rhs.file); - pack(rhs.offset,&buffer[i]); i+=packsize(rhs.offset); + size_t i = 0; + pack( rhs.name, &buffer[i] ); + i += packsize( rhs.name ); + pack( rhs.file, &buffer[i] ); + i += packsize( rhs.file ); + pack( rhs.offset, &buffer[i] ); + i += packsize( rhs.offset ); } template<> -void unpack( IO::DatabaseEntry& data, const char *buffer ) +void unpack( IO::DatabaseEntry &data, const char *buffer ) { - size_t i=0; - unpack(data.name,&buffer[i]); i+=packsize(data.name); - unpack(data.file,&buffer[i]); i+=packsize(data.file); - unpack(data.offset,&buffer[i]); i+=packsize(data.offset); + size_t i = 0; + unpack( data.name, &buffer[i] ); + i += packsize( data.name ); + unpack( data.file, &buffer[i] ); + i += packsize( data.file ); + unpack( data.offset, &buffer[i] ); + i += packsize( data.offset ); } // VariableDatabase template<> -size_t packsize( const IO::VariableDatabase& rhs ) +size_t packsize( const IO::VariableDatabase &rhs ) { - return packsize(rhs.name)+packsize(rhs.type)+packsize(rhs.dim); + return packsize( rhs.name ) + packsize( rhs.type ) + packsize( rhs.dim ); } template<> -void pack( const IO::VariableDatabase& rhs, char *buffer ) +void pack( const IO::VariableDatabase &rhs, char *buffer ) { - size_t i=0; - pack(rhs.name,&buffer[i]); i+=packsize(rhs.name); - pack(rhs.type,&buffer[i]); i+=packsize(rhs.type); - pack(rhs.dim,&buffer[i]); i+=packsize(rhs.dim); + size_t i = 0; + pack( rhs.name, &buffer[i] ); + i += packsize( rhs.name ); + pack( rhs.type, &buffer[i] ); + i += packsize( rhs.type ); + pack( rhs.dim, &buffer[i] ); + i += packsize( rhs.dim ); } template<> -void unpack( IO::VariableDatabase& data, const char *buffer ) +void unpack( IO::VariableDatabase &data, const char *buffer ) { - size_t i=0; - unpack(data.name,&buffer[i]); i+=packsize(data.name); - unpack(data.type,&buffer[i]); i+=packsize(data.type); - unpack(data.dim,&buffer[i]); i+=packsize(data.dim); + size_t i = 0; + unpack( data.name, &buffer[i] ); + i += packsize( data.name ); + unpack( data.type, &buffer[i] ); + i += packsize( data.type ); + unpack( data.dim, &buffer[i] ); + i += packsize( data.dim ); } // MeshDatabase template<> -size_t packsize( const IO::MeshDatabase& data ) +size_t packsize( const IO::MeshDatabase &data ) { - return packsize(data.name) - + packsize(data.type) - + packsize(data.meshClass) - + packsize(data.format) - + packsize(data.domains) - + packsize(data.variables) - + packsize(data.variable_data); + return packsize( data.name ) + packsize( data.type ) + packsize( data.meshClass ) + + packsize( data.format ) + packsize( data.domains ) + packsize( data.variables ) + + packsize( data.variable_data ); } template<> -void pack( const IO::MeshDatabase& rhs, char *buffer ) +void pack( const IO::MeshDatabase &rhs, char *buffer ) { size_t i = 0; - pack(rhs.name,&buffer[i]); i+=packsize(rhs.name); - pack(rhs.type,&buffer[i]); i+=packsize(rhs.type); - pack(rhs.meshClass,&buffer[i]); i+=packsize(rhs.meshClass); - pack(rhs.format,&buffer[i]); i+=packsize(rhs.format); - pack(rhs.domains,&buffer[i]); i+=packsize(rhs.domains); - pack(rhs.variables,&buffer[i]); i+=packsize(rhs.variables); - pack(rhs.variable_data,&buffer[i]); i+=packsize(rhs.variable_data); + pack( rhs.name, &buffer[i] ); + i += packsize( rhs.name ); + pack( rhs.type, &buffer[i] ); + i += packsize( rhs.type ); + pack( rhs.meshClass, &buffer[i] ); + i += packsize( rhs.meshClass ); + pack( rhs.format, &buffer[i] ); + i += packsize( rhs.format ); + pack( rhs.domains, &buffer[i] ); + i += packsize( rhs.domains ); + pack( rhs.variables, &buffer[i] ); + i += packsize( rhs.variables ); + pack( rhs.variable_data, &buffer[i] ); + i += packsize( rhs.variable_data ); } template<> -void unpack( IO::MeshDatabase& data, const char *buffer ) +void unpack( IO::MeshDatabase &data, const char *buffer ) { - size_t i=0; - unpack(data.name,&buffer[i]); i+=packsize(data.name); - unpack(data.type,&buffer[i]); i+=packsize(data.type); - unpack(data.meshClass,&buffer[i]); i+=packsize(data.meshClass); - unpack(data.format,&buffer[i]); i+=packsize(data.format); - unpack(data.domains,&buffer[i]); i+=packsize(data.domains); - unpack(data.variables,&buffer[i]); i+=packsize(data.variables); - unpack(data.variable_data,&buffer[i]); i+=packsize(data.variable_data); + size_t i = 0; + unpack( data.name, &buffer[i] ); + i += packsize( data.name ); + unpack( data.type, &buffer[i] ); + i += packsize( data.type ); + unpack( data.meshClass, &buffer[i] ); + i += packsize( data.meshClass ); + unpack( data.format, &buffer[i] ); + i += packsize( data.format ); + unpack( data.domains, &buffer[i] ); + i += packsize( data.domains ); + unpack( data.variables, &buffer[i] ); + i += packsize( data.variables ); + unpack( data.variable_data, &buffer[i] ); + i += packsize( data.variable_data ); } @@ -132,79 +153,72 @@ namespace IO { /**************************************************** -* VariableDatabase * -****************************************************/ -bool VariableDatabase::operator==(const VariableDatabase& rhs ) const + * VariableDatabase * + ****************************************************/ +bool VariableDatabase::operator==( const VariableDatabase &rhs ) const { - return type==rhs.type && dim==rhs.dim && name==rhs.name; + return type == rhs.type && dim == rhs.dim && name == rhs.name; } -bool VariableDatabase::operator!=(const VariableDatabase& rhs ) const +bool VariableDatabase::operator!=( const VariableDatabase &rhs ) const { - return type!=rhs.type || dim!=rhs.dim || name!=rhs.name; + return type != rhs.type || dim != rhs.dim || name != rhs.name; } -bool VariableDatabase::operator>=(const VariableDatabase& rhs ) const +bool VariableDatabase::operator>=( const VariableDatabase &rhs ) const { - return operator>(rhs) || operator==(rhs); + return operator>( rhs ) || operator==( rhs ); } -bool VariableDatabase::operator<=(const VariableDatabase& rhs ) const +bool VariableDatabase::operator<=( const VariableDatabase &rhs ) const { return !operator>( rhs ); } +bool VariableDatabase::operator>( const VariableDatabase &rhs ) const { - return !operator>(rhs); -} -bool VariableDatabase::operator>(const VariableDatabase& rhs ) const -{ - if ( name>rhs.name ) + if ( name > rhs.name ) return true; - else if ( namerhs.type ) + if ( type > rhs.type ) return true; - else if ( typerhs.dim ) + if ( dim > rhs.dim ) return true; - else if ( dim(rhs) && operator!=(rhs); + return !operator>( rhs ) && operator!=( rhs ); } /**************************************************** -* MeshDatabase * -****************************************************/ -MeshDatabase::MeshDatabase() + * MeshDatabase * + ****************************************************/ +MeshDatabase::MeshDatabase() {} +MeshDatabase::~MeshDatabase() {} +MeshDatabase::MeshDatabase( const MeshDatabase &rhs ) { -} -MeshDatabase::~MeshDatabase() -{ -} -MeshDatabase::MeshDatabase(const MeshDatabase& rhs) -{ - name = rhs.name; - type = rhs.type; - meshClass = rhs.meshClass; - format = rhs.format; - domains = rhs.domains; - variables = rhs.variables; + name = rhs.name; + type = rhs.type; + meshClass = rhs.meshClass; + format = rhs.format; + domains = rhs.domains; + variables = rhs.variables; variable_data = rhs.variable_data; } -MeshDatabase& MeshDatabase::operator=(const MeshDatabase& rhs) +MeshDatabase &MeshDatabase::operator=( const MeshDatabase &rhs ) { - this->name = rhs.name; - this->type = rhs.type; - this->meshClass = rhs.meshClass; - this->format = rhs.format; - this->domains = rhs.domains; - this->variables = rhs.variables; + this->name = rhs.name; + this->type = rhs.type; + this->meshClass = rhs.meshClass; + this->format = rhs.format; + this->domains = rhs.domains; + this->variables = rhs.variables; this->variable_data = rhs.variable_data; return *this; } -VariableDatabase MeshDatabase::getVariableDatabase( const std::string& varname ) const +VariableDatabase MeshDatabase::getVariableDatabase( const std::string &varname ) const { - for (size_t i=0; i list = splitList(line,';'); - name = list[0]; - file = list[1]; - offset = atol(list[2].c_str()); + auto list = splitList( line, ';' ); + name = list[0]; + file = list[1]; + offset = atol( list[2].c_str() ); } -void DatabaseEntry::read( const char* line ) +void DatabaseEntry::read( const char *line ) { - std::vector list = splitList(line,';'); - name = list[0]; - file = list[1]; - offset = atol(list[2].c_str()); + auto list = splitList( line, ';' ); + name = list[0]; + file = list[1]; + offset = atol( list[2].c_str() ); } -void DatabaseEntry::read( const std::string& line ) +void DatabaseEntry::read( const std::string &line ) { - std::vector list = splitList(line.c_str(),';'); - name = list[0]; - file = list[1]; - offset = atol(list[2].c_str()); + auto list = splitList( line.c_str(), ';' ); + name = list[0]; + file = list[1]; + offset = atol( list[2].c_str() ); } // Gather the mesh databases from all processors -inline int tod( int N ) { return (N+7)/sizeof(double); } -std::vector gatherAll( const std::vector& meshes, const Utilities::MPI& comm ) +inline int tod( int N ) { return ( N + 7 ) / sizeof( double ); } +std::vector gatherAll( + const std::vector &meshes, const Utilities::MPI &comm ) { if ( comm.getSize() == 1 ) return meshes; - PROFILE_START("gatherAll"); - PROFILE_START("gatherAll-pack",2); + PROFILE_START( "gatherAll" ); + PROFILE_START( "gatherAll-pack", 2 ); int size = comm.getSize(); // First pack the mesh data to local buffers int localsize = 0; - for (size_t i=0; i data; + PROFILE_START( "gatherAll-unpack", 2 ); + std::map data; pos = 0; while ( pos < globalsize ) { MeshDatabase tmp; - unpack(tmp,(char*)&globalbuf[pos]); - pos += tod(packsize(tmp)); - std::map::iterator it = data.find(tmp.name); - if ( it==data.end() ) { + unpack( tmp, (char *) &globalbuf[pos] ); + pos += tod( packsize( tmp ) ); + std::map::iterator it = data.find( tmp.name ); + if ( it == data.end() ) { data[tmp.name] = tmp; } else { - for (size_t i=0; isecond.domains.push_back(tmp.domains[i]); - for (size_t i=0; isecond.variables.push_back(tmp.variables[i]); - it->second.variable_data.insert(tmp.variable_data.begin(),tmp.variable_data.end()); + for ( size_t i = 0; i < tmp.domains.size(); i++ ) + it->second.domains.push_back( tmp.domains[i] ); + for ( size_t i = 0; i < tmp.variables.size(); i++ ) + it->second.variables.push_back( tmp.variables[i] ); + it->second.variable_data.insert( tmp.variable_data.begin(), tmp.variable_data.end() ); } } - for (auto it=data.begin(); it!=data.end(); ++it) { + for ( auto it = data.begin(); it != data.end(); ++it ) { // Get the unique variables - std::set data2(it->second.variables.begin(),it->second.variables.end()); - it->second.variables = std::vector(data2.begin(),data2.end()); + std::set data2( + it->second.variables.begin(), it->second.variables.end() ); + it->second.variables = std::vector( data2.begin(), data2.end() ); } // Free temporary memory - delete [] localbuf; - delete [] disp; - delete [] globalbuf; + delete[] localbuf; + delete[] disp; + delete[] globalbuf; // Return the results - std::vector data2(data.size()); - size_t i=0; - for (std::map::iterator it=data.begin(); it!=data.end(); ++it, ++i) + std::vector data2( data.size() ); + size_t i = 0; + for ( std::map::iterator it = data.begin(); it != data.end(); + ++it, ++i ) data2[i] = it->second; - PROFILE_STOP("gatherAll-unpack",2); - PROFILE_STOP("gatherAll"); + PROFILE_STOP( "gatherAll-unpack", 2 ); + PROFILE_STOP( "gatherAll" ); return data2; } //! Write the mesh databases to a file -void write( const std::vector& meshes, const std::string& filename ) +void write( const std::vector &meshes, const std::string &filename ) { - PROFILE_START("write"); - FILE *fid = fopen(filename.c_str(),"wb"); - for (size_t i=0; i(meshes[i].type)); - fprintf(fid," meshClass: %s\n",meshes[i].meshClass.c_str()); - fprintf(fid," format: %i\n",static_cast(meshes[i].format)); - for (size_t j=0; j(var.type),var.dim); + PROFILE_START( "write" ); + FILE *fid = fopen( filename.c_str(), "wb" ); + for ( size_t i = 0; i < meshes.size(); i++ ) { + fprintf( fid, "%s\n", meshes[i].name.c_str() ); + fprintf( fid, " type: %i\n", static_cast( meshes[i].type ) ); + fprintf( fid, " meshClass: %s\n", meshes[i].meshClass.c_str() ); + fprintf( fid, " format: %i\n", static_cast( meshes[i].format ) ); + for ( size_t j = 0; j < meshes[i].domains.size(); j++ ) + fprintf( fid, " domain: %s\n", meshes[i].domains[j].write().c_str() ); + fprintf( fid, " variables: " ); + for ( size_t j = 0; j < meshes[i].variables.size(); j++ ) { + const VariableDatabase &var = meshes[i].variables[j]; + fprintf( fid, "%s|%i|%i; ", var.name.c_str(), static_cast( var.type ), var.dim ); } - fprintf(fid,"\n"); - std::map,DatabaseEntry>::const_iterator it; - for (it=meshes[i].variable_data.begin(); it!=meshes[i].variable_data.end(); ++it) { - const char* domain = it->first.first.c_str(); - const char* variable = it->first.second.c_str(); - fprintf(fid," variable(%s,%s): %s\n",domain,variable,it->second.write().c_str()); + fprintf( fid, "\n" ); + std::map, DatabaseEntry>::const_iterator it; + for ( it = meshes[i].variable_data.begin(); it != meshes[i].variable_data.end(); ++it ) { + const char *domain = it->first.first.c_str(); + const char *variable = it->first.second.c_str(); + fprintf( + fid, " variable(%s,%s): %s\n", domain, variable, it->second.write().c_str() ); } } - fclose(fid); - PROFILE_STOP("write"); + fclose( fid ); + PROFILE_STOP( "write" ); } //! Read the mesh databases from a file -std::vector read( const std::string& filename ) +std::vector read( const std::string &filename ) { std::vector meshes; - PROFILE_START("read"); - FILE *fid = fopen(filename.c_str(),"rb"); - if ( fid==NULL ) - ERROR("Error opening file"); + PROFILE_START( "read" ); + FILE *fid = fopen( filename.c_str(), "rb" ); + if ( fid == NULL ) + ERROR( "Error opening file" ); char *line = new char[10000]; - while ( std::fgets(line,1000,fid) != NULL ) { - if ( line[0]<32 ) { + while ( std::fgets( line, 1000, fid ) != NULL ) { + if ( line[0] < 32 ) { // Empty line continue; } else if ( line[0] != ' ' ) { - meshes.resize(meshes.size()+1); - std::string name(line); - name.resize(name.size()-1); + meshes.resize( meshes.size() + 1 ); + std::string name( line ); + name.resize( name.size() - 1 ); meshes.back().name = name; - } else if ( strncmp(line," format:",10)==0 ) { - meshes.back().format = static_cast(atoi(&line[10])); - } else if ( strncmp(line," type:",8)==0 ) { - meshes.back().type = static_cast(atoi(&line[8])); - } else if ( strncmp(line," meshClass:",13)==0 ) { - meshes.back().meshClass = deblank(std::string(&line[13])); - } else if ( strncmp(line," domain:",10)==0 ) { - DatabaseEntry data(&line[10]); - meshes.back().domains.push_back(data); - } else if ( strncmp(line," variables:",13)==0 ) { - MeshDatabase& mesh = meshes.back(); - std::vector variables = splitList(&line[13],';'); - mesh.variables.resize(variables.size()); - for (size_t i=0; i tmp = splitList(variables[i].c_str(),'|'); - ASSERT(tmp.size()==3); + } else if ( strncmp( line, " format:", 10 ) == 0 ) { + meshes.back().format = static_cast( atoi( &line[10] ) ); + } else if ( strncmp( line, " type:", 8 ) == 0 ) { + meshes.back().type = static_cast( atoi( &line[8] ) ); + } else if ( strncmp( line, " meshClass:", 13 ) == 0 ) { + meshes.back().meshClass = deblank( std::string( &line[13] ) ); + } else if ( strncmp( line, " domain:", 10 ) == 0 ) { + DatabaseEntry data( &line[10] ); + meshes.back().domains.push_back( data ); + } else if ( strncmp( line, " variables:", 13 ) == 0 ) { + MeshDatabase &mesh = meshes.back(); + std::vector variables = splitList( &line[13], ';' ); + mesh.variables.resize( variables.size() ); + for ( size_t i = 0; i < variables.size(); i++ ) { + std::vector tmp = splitList( variables[i].c_str(), '|' ); + ASSERT( tmp.size() == 3 ); mesh.variables[i].name = tmp[0]; - mesh.variables[i].type = static_cast(atoi(tmp[1].c_str())); - mesh.variables[i].dim = atoi(tmp[2].c_str()); + mesh.variables[i].type = static_cast( atoi( tmp[1].c_str() ) ); + mesh.variables[i].dim = atoi( tmp[2].c_str() ); } - } else if ( strncmp(line," variable(",12)==0 ) { - size_t i1 = find(line,','); - size_t i2 = find(line,':'); - std::string domain = deblank(std::string(line,12,i1-12)); - std::string variable = deblank(std::string(line,i1+1,i2-i1-2)); - std::pair key(domain,variable); - DatabaseEntry data(&line[i2+1]); - meshes.back().variable_data.insert( - std::pair,DatabaseEntry>(key,data) ); + } else if ( strncmp( line, " variable(", 12 ) == 0 ) { + size_t i1 = find( line, ',' ); + size_t i2 = find( line, ':' ); + std::string domain = deblank( std::string( line, 12, i1 - 12 ) ); + std::string variable = deblank( std::string( line, i1 + 1, i2 - i1 - 2 ) ); + std::pair key( domain, variable ); + DatabaseEntry data( &line[i2 + 1] ); + meshes.back().variable_data.insert( + std::pair, DatabaseEntry>( key, data ) ); } else { - ERROR("Error reading line"); + ERROR( "Error reading line" ); } } - fclose(fid); - delete [] line; - PROFILE_STOP("read"); + fclose( fid ); + delete[] line; + PROFILE_STOP( "read" ); return meshes; } // Return the mesh type -IO::MeshType meshType( const IO::Mesh& mesh ) +IO::MeshType meshType( const IO::Mesh &mesh ) { - IO::MeshType type = IO::Unknown; + IO::MeshType type = IO::MeshType::Unknown; const std::string meshClass = mesh.className(); - if ( meshClass=="PointList" ) { - type = IO::PointMesh; - } else if ( meshClass=="TriList" || meshClass=="TriMesh" ) { - type = IO::SurfaceMesh; - } else if ( meshClass=="DomainMesh" ) { - type = IO::VolumeMesh; + if ( meshClass == "PointList" ) { + type = IO::MeshType::PointMesh; + } else if ( meshClass == "TriList" || meshClass == "TriMesh" ) { + type = IO::MeshType::SurfaceMesh; + } else if ( meshClass == "DomainMesh" ) { + type = IO::MeshType::VolumeMesh; } else { - ERROR("Unknown mesh"); + ERROR( "Unknown mesh" ); } return type; } -} // IO namespace - +} // namespace IO diff --git a/IO/MeshDatabase.h b/IO/MeshDatabase.h index 8e501624..0dfd968c 100644 --- a/IO/MeshDatabase.h +++ b/IO/MeshDatabase.h @@ -1,14 +1,14 @@ #ifndef MeshDatabase_INC #define MeshDatabase_INC -#include "IO/Mesh.h" +#include "IO/Mesh.h" #include "common/MPI.h" #include +#include #include #include #include -#include namespace IO { @@ -17,74 +17,76 @@ class Mesh; //! Enum to identify mesh type -//enum class MeshType : char { PointMesh=1, SurfaceMesh=2, VolumeMesh=3, Unknown=-1 }; -enum MeshType { PointMesh=1, SurfaceMesh=2, VolumeMesh=3, Unknown=-1 }; +// enum class MeshType : char { PointMesh=1, SurfaceMesh=2, VolumeMesh=3, Unknown=-1 }; +enum class MeshType { PointMesh = 1, SurfaceMesh = 2, VolumeMesh = 3, Unknown = -1 }; //! Helper struct for containing offsets for the mesh info struct DatabaseEntry { - std::string name; //!< Name of the entry - std::string file; //!< Name of the file containing the entry - size_t offset; //!< Offset in the file to start reading - std::string write( ) const; //!< Convert the data to a string - void read( const char* line ); //!< Convert the string to data - void read( const std::string& line ); //!< Convert the string to data - DatabaseEntry( ) {} //!< Empty constructor - DatabaseEntry( const char* line ); //!< Convert the string to data - ~DatabaseEntry() {} //!< Destructor + std::string name; //!< Name of the entry + std::string file; //!< Name of the file containing the entry + size_t offset; //!< Offset in the file to start reading + std::string write() const; //!< Convert the data to a string + void read( const char *line ); //!< Convert the string to data + void read( const std::string &line ); //!< Convert the string to data + DatabaseEntry() {} //!< Empty constructor + DatabaseEntry( const char *line ); //!< Convert the string to data + ~DatabaseEntry() {} //!< Destructor }; //! Structure to hold the info about the variables struct VariableDatabase { - std::string name; //!< Name of the variable - IO::VariableType type; //!< Variable - unsigned int dim; //!< Number of points per grid point (1: scalar, 3: vector, ...) + std::string name; //!< Name of the variable + IO::VariableType type; //!< Variable + unsigned int dim; //!< Number of points per grid point (1: scalar, 3: vector, ...) // Overload key operators - bool operator==(const VariableDatabase& rhs ) const; - bool operator!=(const VariableDatabase& rhs ) const; - bool operator>=(const VariableDatabase& rhs ) const; - bool operator<=(const VariableDatabase& rhs ) const; - bool operator> (const VariableDatabase& rhs ) const; - bool operator< (const VariableDatabase& rhs ) const; + bool operator==( const VariableDatabase &rhs ) const; + bool operator!=( const VariableDatabase &rhs ) const; + bool operator>=( const VariableDatabase &rhs ) const; + bool operator<=( const VariableDatabase &rhs ) const; + bool operator>( const VariableDatabase &rhs ) const; + bool operator<( const VariableDatabase &rhs ) const; }; //! Structure to hold the info about the meshes struct MeshDatabase { - typedef std::pair variable_id; + typedef std::pair variable_id; std::string name; //!< Name of the mesh MeshType type; //!< Mesh type std::string meshClass; //!< Mesh class unsigned char format; //!< Data format (1: old, 2: new, 3: new (single), 4: silo) std::vector domains; //!< List of the domains - std::vector variables; //!< List of the variables - std::map variable_data; //!< Data for the variables - VariableDatabase getVariableDatabase( const std::string& varname ) const; + std::vector variables; //!< List of the variables + std::map variable_data; //!< Data for the variables + VariableDatabase getVariableDatabase( const std::string &varname ) const; + public: MeshDatabase(); ~MeshDatabase(); - MeshDatabase(const MeshDatabase&); - MeshDatabase& operator=(const MeshDatabase&); + MeshDatabase( const MeshDatabase & ); + MeshDatabase &operator=( const MeshDatabase & ); }; //! Gather the mesh databases from all processors -std::vector gatherAll( const std::vector& meshes, const Utilities::MPI& comm ); +std::vector gatherAll( + const std::vector &meshes, const Utilities::MPI &comm ); //! Write the mesh databases to a file -void write( const std::vector& meshes, const std::string& filename ); +void write( const std::vector &meshes, const std::string &filename ); //! Read the mesh databases from a file -std::vector read( const std::string& filename ); +std::vector read( const std::string &filename ); //! Return the mesh type -IO::MeshType meshType( const IO::Mesh& mesh ); +IO::MeshType meshType( const IO::Mesh &mesh ); -} // IO namespace +} // namespace IO #endif diff --git a/IO/PIO.cpp b/IO/PIO.cpp index fe0f7db4..f959cb49 100644 --- a/IO/PIO.cpp +++ b/IO/PIO.cpp @@ -1,10 +1,10 @@ #include "IO/PIO.h" -#include "common/Utilities.h" #include "common/MPI.h" +#include "common/Utilities.h" +#include #include #include -#include namespace IO { @@ -15,19 +15,18 @@ static ParallelStreamBuffer perr_buffer; static ParallelStreamBuffer plog_buffer; -std::ostream pout(&pout_buffer); -std::ostream perr(&perr_buffer); -std::ostream plog(&plog_buffer); - +std::ostream pout( &pout_buffer ); +std::ostream perr( &perr_buffer ); +std::ostream plog( &plog_buffer ); /**************************************************************************** -* Functions to control logging * -****************************************************************************/ -std::ofstream *global_filestream=NULL; -static void shutdownFilestream( ) + * Functions to control logging * + ****************************************************************************/ +std::ofstream *global_filestream = NULL; +static void shutdownFilestream() { - if ( global_filestream!=NULL ) { + if ( global_filestream != NULL ) { global_filestream->flush(); global_filestream->close(); delete global_filestream; @@ -37,16 +36,16 @@ static void shutdownFilestream( ) void Utilities::logOnlyNodeZero( const std::string &filename ) { int rank = 0; - #ifdef USE_MPI - MPI_Comm_rank( MPI_COMM_WORLD, &rank ); - #endif +#ifdef USE_MPI + MPI_Comm_rank( MPI_COMM_WORLD, &rank ); +#endif if ( rank == 0 ) - logAllNodes(filename,true); + logAllNodes( filename, true ); } void Utilities::logAllNodes( const std::string &filename, bool singleStream ) { if ( singleStream ) - ERROR("Not implimented yet"); + ERROR( "Not implimented yet" ); // If the filestream was open, then close it and reset streams shutdownFilestream(); @@ -55,33 +54,33 @@ void Utilities::logAllNodes( const std::string &filename, bool singleStream ) std::string full_filename = filename; if ( !singleStream ) { int rank = 0; - #ifdef USE_MPI - MPI_Comm_rank( MPI_COMM_WORLD, &rank ); - #endif +#ifdef USE_MPI + MPI_Comm_rank( MPI_COMM_WORLD, &rank ); +#endif char tmp[100]; - sprintf(tmp,".%04i",rank); - full_filename += std::string(tmp); + sprintf( tmp, ".%04i", rank ); + full_filename += std::string( tmp ); } - global_filestream = new std::ofstream(full_filename.c_str()); + global_filestream = new std::ofstream( full_filename.c_str() ); - if ( !(*global_filestream) ) { + if ( !( *global_filestream ) ) { delete global_filestream; global_filestream = NULL; perr << "PIO: Could not open log file ``" << full_filename << "''\n"; } else { - pout_buffer.setOutputStream(global_filestream); - pout_buffer.setOutputStream(&std::cout); - perr_buffer.setOutputStream(global_filestream); - perr_buffer.setOutputStream(&std::cerr); - plog_buffer.setOutputStream(global_filestream); + pout_buffer.setOutputStream( global_filestream ); + pout_buffer.setOutputStream( &std::cout ); + perr_buffer.setOutputStream( global_filestream ); + perr_buffer.setOutputStream( &std::cerr ); + plog_buffer.setOutputStream( global_filestream ); } } /**************************************************************************** -* ParallelStreamBuffer class * -****************************************************************************/ -void Utilities::stopLogging( ) + * ParallelStreamBuffer class * + ****************************************************************************/ +void Utilities::stopLogging() { pout_buffer.reset(); perr_buffer.reset(); @@ -93,77 +92,71 @@ void Utilities::stopLogging( ) /**************************************************************************** -* ParallelStreamBuffer class * -****************************************************************************/ -ParallelStreamBuffer::ParallelStreamBuffer( ): - d_rank(0), d_size(0), d_buffer_size(0), d_buffer(NULL) + * ParallelStreamBuffer class * + ****************************************************************************/ +ParallelStreamBuffer::ParallelStreamBuffer() + : d_rank( 0 ), d_size( 0 ), d_buffer_size( 0 ), d_buffer( NULL ) { } -ParallelStreamBuffer:: ~ParallelStreamBuffer() -{ - delete [] d_buffer; -} -void ParallelStreamBuffer::setOutputStream( std::ostream *stream ) -{ - d_stream.push_back( stream ); -} +ParallelStreamBuffer::~ParallelStreamBuffer() { delete[] d_buffer; } +void ParallelStreamBuffer::setOutputStream( std::ostream *stream ) { d_stream.push_back( stream ); } int ParallelStreamBuffer::sync() { - for (size_t i=0; i d_buffer_size ) { - if ( d_buffer_size==0 ) { + if ( d_buffer_size == 0 ) { d_buffer_size = 1024; - d_buffer = new char[d_buffer_size]; - memset(d_buffer,0,d_buffer_size); + d_buffer = new char[d_buffer_size]; + memset( d_buffer, 0, d_buffer_size ); } while ( size > d_buffer_size ) { char *tmp = d_buffer; d_buffer_size *= 2; d_buffer = new char[d_buffer_size]; - memset(d_buffer,0,d_buffer_size); - memcpy(d_buffer,tmp,d_size); - delete [] tmp; + memset( d_buffer, 0, d_buffer_size ); + memcpy( d_buffer, tmp, d_size ); + delete[] tmp; } } } -std::streamsize ParallelStreamBuffer::xsputn( const char* text, std::streamsize n ) +std::streamsize ParallelStreamBuffer::xsputn( const char *text, std::streamsize n ) { - reserve(d_size+n); - memcpy(&d_buffer[d_size],text,n); + reserve( d_size + n ); + memcpy( &d_buffer[d_size], text, n ); d_size += n; - if ( text[n-1]==0 || text[n-1]==10 ) { sync(); } + if ( text[n - 1] == 0 || text[n - 1] == 10 ) { + sync(); + } return n; } -int ParallelStreamBuffer::overflow(int ch) +int ParallelStreamBuffer::overflow( int ch ) { - reserve(d_size+1); + reserve( d_size + 1 ); d_buffer[d_size] = ch; d_size++; - if ( ch==0 || ch==10 ) { sync(); } - return std::char_traits::to_int_type(ch); + if ( ch == 0 || ch == 10 ) { + sync(); + } + return std::char_traits::to_int_type( ch ); } -int ParallelStreamBuffer::underflow() -{ - return -1; -} -void ParallelStreamBuffer::reset() +int ParallelStreamBuffer::underflow() { return -1; } +void ParallelStreamBuffer::reset() { sync(); d_stream.clear(); - delete [] d_buffer; - d_buffer = NULL; + delete[] d_buffer; + d_buffer = NULL; d_buffer_size = 0; } -} // IO namespace - +} // namespace IO diff --git a/IO/PIO.h b/IO/PIO.h index b6d8b103..9b8aeb89 100644 --- a/IO/PIO.h +++ b/IO/PIO.h @@ -17,7 +17,7 @@ extern std::ostream pout; /*! * Parallel output stream perr writes to the standard error from all nodes. - * Output is prepended with the processor number. + * Output is prepended with the processor number. */ extern std::ostream perr; @@ -45,12 +45,11 @@ inline int printp( const char *format, ... ); class ParallelStreamBuffer : public std::streambuf { public: - /*! * Create a parallel buffer class. The object will require further * initialization to set up the I/O streams and prefix string. */ - ParallelStreamBuffer( ); + ParallelStreamBuffer(); /*! * Set the output file stream (multiple output streams are supported) @@ -60,26 +59,26 @@ public: /*! * The destructor simply deallocates any internal data - * buffers. It does not modify the output streams. + * buffers. It does not modify the output streams. */ virtual ~ParallelStreamBuffer(); /*! * Synchronize the parallel buffer (called from streambuf). */ - virtual int sync(); + virtual int sync(); /** * Write the specified number of characters into the output stream (called * from streambuf). - */ - virtual std::streamsize xsputn(const char* text, std::streamsize n); + */ + virtual std::streamsize xsputn( const char *text, std::streamsize n ); /*! * Write an overflow character into the parallel buffer (called from * streambuf). */ - virtual int overflow(int ch); + virtual int overflow( int ch ); /*! * Read an overflow character from the parallel buffer (called from @@ -98,30 +97,30 @@ private: size_t d_size; size_t d_buffer_size; char *d_buffer; - std::vector d_stream; + std::vector d_stream; inline void reserve( size_t size ); }; namespace Utilities { - /*! - * Log messages for node zero only to the specified filename. All output - * to pout, perr, and plog on node zero will go to the log file. - */ - void logOnlyNodeZero( const std::string &filename ); +/*! + * Log messages for node zero only to the specified filename. All output + * to pout, perr, and plog on node zero will go to the log file. + */ +void logOnlyNodeZero( const std::string &filename ); - /*! - * Log messages from all nodes. The diagnostic data for processor XXXXX - * will be sent to a file with the name filename.XXXXX, where filename is - * the function argument. - */ - void logAllNodes( const std::string &filename, bool singleStream=false ); +/*! + * Log messages from all nodes. The diagnostic data for processor XXXXX + * will be sent to a file with the name filename.XXXXX, where filename is + * the function argument. + */ +void logAllNodes( const std::string &filename, bool singleStream = false ); - /*! - * Stop logging messages, flush buffers, and reset memory. - */ - void stopLogging( ); +/*! + * Stop logging messages, flush buffers, and reset memory. + */ +void stopLogging(); } // namespace Utilities diff --git a/IO/PIO.hpp b/IO/PIO.hpp index 67b32cdb..748bf32b 100644 --- a/IO/PIO.hpp +++ b/IO/PIO.hpp @@ -3,9 +3,9 @@ #include "IO/PIO.h" +#include #include #include -#include namespace IO { @@ -13,17 +13,17 @@ namespace IO { inline int printp( const char *format, ... ) { - va_list ap; - va_start(ap,format); + va_list ap; + va_start( ap, format ); char tmp[1024]; - int n = vsprintf(tmp,format,ap); - va_end(ap); + int n = vsprintf( tmp, format, ap ); + va_end( ap ); pout << tmp; pout.flush(); return n; } -} // IO namespace +} // namespace IO #endif diff --git a/IO/PackData.cpp b/IO/PackData.cpp index f10d9ca7..3782914c 100644 --- a/IO/PackData.cpp +++ b/IO/PackData.cpp @@ -4,102 +4,101 @@ /******************************************************** -* Concrete implimentations for packing/unpacking * -********************************************************/ + * Concrete implimentations for packing/unpacking * + ********************************************************/ // unsigned char template<> -size_t packsize( const unsigned char& rhs ) +size_t packsize( const unsigned char &rhs ) { - return sizeof(unsigned char); + return sizeof( unsigned char ); } template<> -void pack( const unsigned char& rhs, char *buffer ) +void pack( const unsigned char &rhs, char *buffer ) { - memcpy(buffer,&rhs,sizeof(unsigned char)); + memcpy( buffer, &rhs, sizeof( unsigned char ) ); } template<> -void unpack( unsigned char& data, const char *buffer ) +void unpack( unsigned char &data, const char *buffer ) { - memcpy(&data,buffer,sizeof(unsigned char)); + memcpy( &data, buffer, sizeof( unsigned char ) ); } // char template<> -size_t packsize( const char& rhs ) +size_t packsize( const char &rhs ) { - return sizeof(char); + return sizeof( char ); } template<> -void pack( const char& rhs, char *buffer ) +void pack( const char &rhs, char *buffer ) { - memcpy(buffer,&rhs,sizeof(char)); + memcpy( buffer, &rhs, sizeof( char ) ); } template<> -void unpack( char& data, const char *buffer ) +void unpack( char &data, const char *buffer ) { - memcpy(&data,buffer,sizeof(char)); + memcpy( &data, buffer, sizeof( char ) ); } // int template<> -size_t packsize( const int& rhs ) +size_t packsize( const int &rhs ) { - return sizeof(int); + return sizeof( int ); } template<> -void pack( const int& rhs, char *buffer ) +void pack( const int &rhs, char *buffer ) { - memcpy(buffer,&rhs,sizeof(int)); + memcpy( buffer, &rhs, sizeof( int ) ); } template<> -void unpack( int& data, const char *buffer ) +void unpack( int &data, const char *buffer ) { - memcpy(&data,buffer,sizeof(int)); + memcpy( &data, buffer, sizeof( int ) ); } // unsigned int template<> -size_t packsize( const unsigned int& rhs ) +size_t packsize( const unsigned int &rhs ) { - return sizeof(unsigned int); + return sizeof( unsigned int ); } template<> -void pack( const unsigned int& rhs, char *buffer ) +void pack( const unsigned int &rhs, char *buffer ) { - memcpy(buffer,&rhs,sizeof(int)); + memcpy( buffer, &rhs, sizeof( int ) ); } template<> -void unpack( unsigned int& data, const char *buffer ) +void unpack( unsigned int &data, const char *buffer ) { - memcpy(&data,buffer,sizeof(int)); + memcpy( &data, buffer, sizeof( int ) ); } // size_t template<> -size_t packsize( const size_t& rhs ) +size_t packsize( const size_t &rhs ) { - return sizeof(size_t); + return sizeof( size_t ); } template<> -void pack( const size_t& rhs, char *buffer ) +void pack( const size_t &rhs, char *buffer ) { - memcpy(buffer,&rhs,sizeof(size_t)); + memcpy( buffer, &rhs, sizeof( size_t ) ); } template<> -void unpack( size_t& data, const char *buffer ) +void unpack( size_t &data, const char *buffer ) { - memcpy(&data,buffer,sizeof(size_t)); + memcpy( &data, buffer, sizeof( size_t ) ); } // std::string template<> -size_t packsize( const std::string& rhs ) +size_t packsize( const std::string &rhs ) { - return rhs.size()+1; + return rhs.size() + 1; } template<> -void pack( const std::string& rhs, char *buffer ) +void pack( const std::string &rhs, char *buffer ) { - memcpy(buffer,rhs.c_str(),rhs.size()+1); + memcpy( buffer, rhs.c_str(), rhs.size() + 1 ); } template<> -void unpack( std::string& data, const char *buffer ) +void unpack( std::string &data, const char *buffer ) { - data = std::string(buffer); + data = std::string( buffer ); } - diff --git a/IO/PackData.h b/IO/PackData.h index 85326c0b..f7c1d748 100644 --- a/IO/PackData.h +++ b/IO/PackData.h @@ -2,77 +2,76 @@ #ifndef included_PackData #define included_PackData -#include -#include #include +#include +#include //! Template function to return the buffer size required to pack a class template -size_t packsize( const TYPE& rhs ); +size_t packsize( const TYPE &rhs ); //! Template function to pack a class to a buffer template -void pack( const TYPE& rhs, char *buffer ); +void pack( const TYPE &rhs, char *buffer ); //! Template function to unpack a class from a buffer template -void unpack( TYPE& data, const char *buffer ); +void unpack( TYPE &data, const char *buffer ); //! Template function to return the buffer size required to pack a std::vector template -size_t packsize( const std::vector& rhs ); +size_t packsize( const std::vector &rhs ); //! Template function to pack a class to a buffer template -void pack( const std::vector& rhs, char *buffer ); +void pack( const std::vector &rhs, char *buffer ); //! Template function to pack a class to a buffer template -void unpack( std::vector& data, const char *buffer ); +void unpack( std::vector &data, const char *buffer ); //! Template function to return the buffer size required to pack a std::pair template -size_t packsize( const std::pair& rhs ); +size_t packsize( const std::pair &rhs ); //! Template function to pack a class to a buffer template -void pack( const std::pair& rhs, char *buffer ); +void pack( const std::pair &rhs, char *buffer ); //! Template function to pack a class to a buffer template -void unpack( std::pair& data, const char *buffer ); +void unpack( std::pair &data, const char *buffer ); //! Template function to return the buffer size required to pack a std::map template -size_t packsize( const std::map& rhs ); +size_t packsize( const std::map &rhs ); //! Template function to pack a class to a buffer template -void pack( const std::map& rhs, char *buffer ); +void pack( const std::map &rhs, char *buffer ); //! Template function to pack a class to a buffer template -void unpack( std::map& data, const char *buffer ); +void unpack( std::map &data, const char *buffer ); //! Template function to return the buffer size required to pack a std::set template -size_t packsize( const std::set& rhs ); +size_t packsize( const std::set &rhs ); //! Template function to pack a class to a buffer template -void pack( const std::set& rhs, char *buffer ); +void pack( const std::set &rhs, char *buffer ); //! Template function to pack a class to a buffer template -void unpack( std::set& data, const char *buffer ); +void unpack( std::set &data, const char *buffer ); #include "IO/PackData.hpp" #endif - diff --git a/IO/PackData.hpp b/IO/PackData.hpp index 006cdf73..fd74aa64 100644 --- a/IO/PackData.hpp +++ b/IO/PackData.hpp @@ -4,152 +4,156 @@ #include "IO/PackData.h" +#include +#include #include #include -#include -#include - /******************************************************** -* Default instantiations for std::vector * -********************************************************/ + * Default instantiations for std::vector * + ********************************************************/ template -size_t packsize( const std::vector& rhs ) +size_t packsize( const std::vector &rhs ) { - size_t bytes = sizeof(size_t); - for (size_t i=0; i -void pack( const std::vector& rhs, char *buffer ) +void pack( const std::vector &rhs, char *buffer ) { size_t size = rhs.size(); - memcpy(buffer,&size,sizeof(size_t)); - size_t pos = sizeof(size_t); - for (size_t i=0; i -void unpack( std::vector& data, const char *buffer ) +void unpack( std::vector &data, const char *buffer ) { size_t size; - memcpy(&size,buffer,sizeof(size_t)); + memcpy( &size, buffer, sizeof( size_t ) ); data.clear(); - data.resize(size); - size_t pos = sizeof(size_t); - for (size_t i=0; i -size_t packsize( const std::pair& rhs ) +size_t packsize( const std::pair &rhs ) { - return packsize(rhs.first)+packsize(rhs.second); + return packsize( rhs.first ) + packsize( rhs.second ); } template -void pack( const std::pair& rhs, char *buffer ) +void pack( const std::pair &rhs, char *buffer ) { - pack(rhs.first,buffer); - pack(rhs.second,&buffer[packsize(rhs.first)]); + pack( rhs.first, buffer ); + pack( rhs.second, &buffer[packsize( rhs.first )] ); } template -void unpack( std::pair& data, const char *buffer ) +void unpack( std::pair &data, const char *buffer ) { - unpack(data.first,buffer); - unpack(data.second,&buffer[packsize(data.first)]); + unpack( data.first, buffer ); + unpack( data.second, &buffer[packsize( data.first )] ); } /******************************************************** -* Default instantiations for std::map * -********************************************************/ + * Default instantiations for std::map * + ********************************************************/ template -size_t packsize( const std::map& rhs ) +size_t packsize( const std::map &rhs ) { - size_t bytes = sizeof(size_t); - typename std::map::const_iterator it; - for (it=rhs.begin(); it!=rhs.end(); ++it) { - bytes += packsize(it->first); - bytes += packsize(it->second); + size_t bytes = sizeof( size_t ); + typename std::map::const_iterator it; + for ( it = rhs.begin(); it != rhs.end(); ++it ) { + bytes += packsize( it->first ); + bytes += packsize( it->second ); } return bytes; } template -void pack( const std::map& rhs, char *buffer ) +void pack( const std::map &rhs, char *buffer ) { size_t N = rhs.size(); - pack(N,buffer); - size_t pos = sizeof(size_t); - typename std::map::const_iterator it; - for (it=rhs.begin(); it!=rhs.end(); ++it) { - pack(it->first,&buffer[pos]); pos+=packsize(it->first); - pack(it->second,&buffer[pos]); pos+=packsize(it->second); + pack( N, buffer ); + size_t pos = sizeof( size_t ); + typename std::map::const_iterator it; + for ( it = rhs.begin(); it != rhs.end(); ++it ) { + pack( it->first, &buffer[pos] ); + pos += packsize( it->first ); + pack( it->second, &buffer[pos] ); + pos += packsize( it->second ); } } template -void unpack( std::map& data, const char *buffer ) +void unpack( std::map &data, const char *buffer ) { size_t N = 0; - unpack(N,buffer); - size_t pos = sizeof(size_t); + unpack( N, buffer ); + size_t pos = sizeof( size_t ); data.clear(); - for (size_t i=0; i tmp; - unpack(tmp.first,&buffer[pos]); pos+=packsize(tmp.first); - unpack(tmp.second,&buffer[pos]); pos+=packsize(tmp.second); - data.insert(tmp); + for ( size_t i = 0; i < N; i++ ) { + std::pair tmp; + unpack( tmp.first, &buffer[pos] ); + pos += packsize( tmp.first ); + unpack( tmp.second, &buffer[pos] ); + pos += packsize( tmp.second ); + data.insert( tmp ); } } /******************************************************** -* Default instantiations for std::set * -********************************************************/ + * Default instantiations for std::set * + ********************************************************/ template -size_t packsize( const std::set& rhs ) +size_t packsize( const std::set &rhs ) { - size_t bytes = sizeof(size_t); + size_t bytes = sizeof( size_t ); typename std::set::const_iterator it; - for (it=rhs.begin(); it!=rhs.end(); ++it) { - bytes += packsize(*it); + for ( it = rhs.begin(); it != rhs.end(); ++it ) { + bytes += packsize( *it ); } return bytes; } template -void pack( const std::set& rhs, char *buffer ) +void pack( const std::set &rhs, char *buffer ) { size_t N = rhs.size(); - pack(N,buffer); - size_t pos = sizeof(size_t); + pack( N, buffer ); + size_t pos = sizeof( size_t ); typename std::set::const_iterator it; - for (it=rhs.begin(); it!=rhs.end(); ++it) { - pack(*it); pos+=packsize(*it); + for ( it = rhs.begin(); it != rhs.end(); ++it ) { + pack( *it ); + pos += packsize( *it ); } } template -void unpack( std::set& data, const char *buffer ) +void unpack( std::set &data, const char *buffer ) { size_t N = 0; - unpack(N,buffer); - size_t pos = sizeof(size_t); + unpack( N, buffer ); + size_t pos = sizeof( size_t ); data.clear(); - for (size_t i=0; i +#include +#include #include +#include #include #include -#include -#include // Inline function to read line without a return argument -static inline void fgetl( char * str, int num, FILE * stream ) +static inline void fgetl( char *str, int num, FILE *stream ) { - char* ptr = fgets( str, num, stream ); - if ( 0 ) {char *temp = (char *)&ptr; temp++;} + char *ptr = fgets( str, num, stream ); + if ( 0 ) { + char *temp = (char *) &ptr; + temp++; + } +} + + +// Check if the file exists +bool fileExists( const std::string &filename ) +{ + std::ifstream ifile( filename.c_str() ); + return ifile.good(); } // Get the path to a file -std::string IO::getPath( const std::string& filename ) +std::string IO::getPath( const std::string &filename ) { - std::string file(filename); - size_t k1 = file.rfind(47); - size_t k2 = file.rfind(92); - if ( k1==std::string::npos ) { k1=0; } - if ( k2==std::string::npos ) { k2=0; } - return file.substr(0,std::max(k1,k2)); + std::string file( filename ); + size_t k1 = file.rfind( 47 ); + size_t k2 = file.rfind( 92 ); + if ( k1 == std::string::npos ) { + k1 = 0; + } + if ( k2 == std::string::npos ) { + k2 = 0; + } + return file.substr( 0, std::max( k1, k2 ) ); } // List the timesteps in the given directory (dumps.LBPM) -std::vector IO::readTimesteps( const std::string& path, const std::string& format ) +std::vector IO::readTimesteps( const std::string &path, const std::string &format ) { // Get the name of the summary filename std::string filename = path + "/"; - if ( format=="old" || format=="new" ) + if ( format == "old" || format == "new" ) { filename += "summary.LBM"; - else if ( format=="silo" ) + } else if ( format == "silo" ) { filename += "LBM.visit"; - else + } else if ( format == "auto" ) { + bool test_old = fileExists( path + "/summary.LBM" ); + bool test_silo = fileExists( path + "/LBM.visit" ); + if ( test_old && test_silo ) { + ERROR( "Unable to determine format (both summary.LBM and LBM.visit exist)" ); + } else if ( test_old ) { + filename += "summary.LBM"; + } else if ( test_silo ) { + filename += "LBM.visit"; + } else { + ERROR( "Unable to determine format (neither summary.LBM or LBM.visit exist)" ); + } + } else { ERROR( "Unknown format: " + format ); - PROFILE_START("readTimesteps"); + } + PROFILE_START( "readTimesteps" ); // Read the data - FILE *fid= fopen(filename.c_str(),"rb"); - if ( fid==NULL ) - ERROR("Error opening file"); + FILE *fid = fopen( filename.c_str(), "rb" ); + if ( fid == NULL ) + ERROR( "Error opening file" ); std::vector timesteps; char buf[1000]; - while (fgets(buf,sizeof(buf),fid) != NULL) { - std::string line(buf); - line.resize(line.size()-1); + while ( fgets( buf, sizeof( buf ), fid ) != NULL ) { + std::string line( buf ); + line.resize( line.size() - 1 ); auto pos = line.find( "summary.silo" ); if ( pos != std::string::npos ) - line.resize(pos); + line.resize( pos ); if ( line.empty() ) continue; - timesteps.push_back(line); + timesteps.push_back( line ); } - fclose(fid); - PROFILE_STOP("readTimesteps"); + fclose( fid ); + PROFILE_STOP( "readTimesteps" ); return timesteps; return timesteps; } +// Get the maximum number of domains +int IO::maxDomains( const std::string &path, const std::string &format, const Utilities::MPI &comm ) +{ + int rank = comm.getRank(); + int n_domains = 0; + if ( rank == 0 ) { + // Get the timesteps + auto timesteps = IO::readTimesteps( path, format ); + ASSERT( !timesteps.empty() ); + // Get the database for the first domain + auto db = IO::getMeshList( path, timesteps[0] ); + for ( size_t i = 0; i < db.size(); i++ ) + n_domains = std::max( n_domains, db[i].domains.size() ); + } + return comm.bcast( n_domains, 0 ); +} + + // Read the data for the given timestep -std::vector IO::readData( const std::string& path, const std::string& timestep, const Utilities::MPI &comm ) +std::vector IO::readData( + const std::string &path, const std::string ×tep, int rank ) { // Get the mesh databases auto db = IO::getMeshList( path, timestep ); // Create the data - std::vector data( db .size() ); - for ( size_t i=0; i data( db.size() ); + for ( size_t i = 0; i < data.size(); i++ ) { data[i].precision = IO::DataType::Double; - data[i].meshName = db [i].name; - data[i].mesh = getMesh( path, timestep, db [i], domain ); + data[i].meshName = db[i].name; + data[i].mesh = getMesh( path, timestep, db[i], rank ); data[i].vars.resize( db[i].variables.size() ); - for ( size_t j=0; j IO::getMeshList( const std::string& path, const std::string& timestep ) +std::vector IO::getMeshList( + const std::string &path, const std::string ×tep ) { std::string filename = path + "/" + timestep + "/LBM.summary"; return IO::read( filename ); @@ -103,270 +150,271 @@ std::vector IO::getMeshList( const std::string& path, const st // Read the given mesh domain -std::shared_ptr IO::getMesh( const std::string& path, const std::string& timestep, - const IO::MeshDatabase& meshDatabase, int domain ) +std::shared_ptr IO::getMesh( const std::string &path, const std::string ×tep, + const IO::MeshDatabase &meshDatabase, int domain ) { - PROFILE_START("getMesh"); + PROFILE_START( "getMesh" ); std::shared_ptr mesh; - if ( meshDatabase.format==1 ) { + if ( meshDatabase.format == 1 ) { // Old format (binary doubles) std::string filename = path + "/" + timestep + "/" + meshDatabase.domains[domain].file; - FILE *fid = fopen(filename.c_str(),"rb"); - INSIST(fid!=NULL,"Error opening file"); + FILE *fid = fopen( filename.c_str(), "rb" ); + INSIST( fid != NULL, "Error opening file" ); fseek( fid, 0, SEEK_END ); - size_t bytes = ftell(fid); - size_t N_max = bytes/sizeof(double)+1000; + size_t bytes = ftell( fid ); + size_t N_max = bytes / sizeof( double ) + 1000; double *data = new double[N_max]; - fseek(fid,0,SEEK_SET); - size_t count = fread(data,sizeof(double),N_max,fid); - fclose(fid); - if ( count%3 != 0 ) - ERROR("Error reading file"); - if ( meshDatabase.type==IO::PointMesh ) { - size_t N = count/3; - std::shared_ptr pointlist( new PointList(N) ); - std::vector& P = pointlist->points; - for (size_t i=0; i pointlist( new PointList( N ) ); + std::vector &P = pointlist->points; + for ( size_t i = 0; i < N; i++ ) { + P[i].x = data[3 * i + 0]; + P[i].y = data[3 * i + 1]; + P[i].z = data[3 * i + 2]; } mesh = pointlist; - } else if ( meshDatabase.type==IO::SurfaceMesh ) { - if ( count%9 != 0 ) - ERROR("Error reading file (2)"); - size_t N_tri = count/9; - std::shared_ptr trilist( new TriList(N_tri) ); - std::vector& A = trilist->A; - std::vector& B = trilist->B; - std::vector& C = trilist->C; - for (size_t i=0; i trilist( new TriList( N_tri ) ); + std::vector &A = trilist->A; + std::vector &B = trilist->B; + std::vector &C = trilist->C; + for ( size_t i = 0; i < N_tri; i++ ) { + A[i].x = data[9 * i + 0]; + A[i].y = data[9 * i + 1]; + A[i].z = data[9 * i + 2]; + B[i].x = data[9 * i + 3]; + B[i].y = data[9 * i + 4]; + B[i].z = data[9 * i + 5]; + C[i].x = data[9 * i + 6]; + C[i].y = data[9 * i + 7]; + C[i].z = data[9 * i + 8]; } mesh = trilist; - } else if ( meshDatabase.type==IO::VolumeMesh ) { + } else if ( meshDatabase.type == IO::MeshType::VolumeMesh ) { // this was never supported in the old format mesh = std::shared_ptr( new DomainMesh() ); } else { - ERROR("Unknown mesh type"); + ERROR( "Unknown mesh type" ); } - delete [] data; - } else if ( meshDatabase.format==2 ) { - const DatabaseEntry& database = meshDatabase.domains[domain]; - std::string filename = path + "/" + timestep + "/" + database.file; - FILE *fid = fopen(filename.c_str(),"rb"); - fseek(fid,database.offset,SEEK_SET); + delete[] data; + } else if ( meshDatabase.format == 2 ) { + const DatabaseEntry &database = meshDatabase.domains[domain]; + std::string filename = path + "/" + timestep + "/" + database.file; + FILE *fid = fopen( filename.c_str(), "rb" ); + fseek( fid, database.offset, SEEK_SET ); char line[1000]; - fgetl(line,1000,fid); - size_t i1 = find(line,':'); - size_t i2 = find(&line[i1+1],':')+i1+1; - size_t bytes = atol(&line[i2+1]); - char *data = new char[bytes]; - size_t count = fread(data,1,bytes,fid); - fclose(fid); - ASSERT(count==bytes); - if ( meshDatabase.meshClass=="PointList" ) { + fgetl( line, 1000, fid ); + size_t i1 = find( line, ':' ); + size_t i2 = find( &line[i1 + 1], ':' ) + i1 + 1; + size_t bytes = atol( &line[i2 + 1] ); + char *data = new char[bytes]; + size_t count = fread( data, 1, bytes, fid ); + fclose( fid ); + ASSERT( count == bytes ); + if ( meshDatabase.meshClass == "PointList" ) { mesh.reset( new IO::PointList() ); - } else if ( meshDatabase.meshClass=="TriMesh" ) { + } else if ( meshDatabase.meshClass == "TriMesh" ) { mesh.reset( new IO::TriMesh() ); - } else if ( meshDatabase.meshClass=="TriList" ) { + } else if ( meshDatabase.meshClass == "TriList" ) { mesh.reset( new IO::TriList() ); - } else if ( meshDatabase.meshClass=="DomainMesh" ) { + } else if ( meshDatabase.meshClass == "DomainMesh" ) { mesh.reset( new IO::DomainMesh() ); } else { - ERROR("Unknown mesh class"); + ERROR( "Unknown mesh class" ); } - mesh->unpack( std::pair(bytes,data) ); - delete [] data; - } else if ( meshDatabase.format==4 ) { + mesh->unpack( std::pair( bytes, data ) ); + delete[] data; + } else if ( meshDatabase.format == 4 ) { // Reading a silo file #ifdef USE_SILO - const DatabaseEntry& database = meshDatabase.domains[domain]; - std::string filename = path + "/" + timestep + "/" + database.file; - auto fid = silo::open( filename, silo::READ ); - if ( meshDatabase.meshClass=="PointList" ) { + const DatabaseEntry &database = meshDatabase.domains[domain]; + std::string filename = path + "/" + timestep + "/" + database.file; + auto fid = silo::open( filename, silo::READ ); + if ( meshDatabase.meshClass == "PointList" ) { Array coords = silo::readPointMesh( fid, database.name ); - ASSERT(coords.size(1)==3); - std::shared_ptr mesh2( new IO::PointList( coords.size(0) ) ); - for (size_t i=0; ipoints[i].x = coords(i,0); - mesh2->points[i].y = coords(i,1); - mesh2->points[i].z = coords(i,2); + ASSERT( coords.size( 1 ) == 3 ); + std::shared_ptr mesh2( new IO::PointList( coords.size( 0 ) ) ); + for ( size_t i = 0; i < coords.size( 1 ); i++ ) { + mesh2->points[i].x = coords( i, 0 ); + mesh2->points[i].y = coords( i, 1 ); + mesh2->points[i].z = coords( i, 2 ); } mesh = mesh2; - } else if ( meshDatabase.meshClass=="TriMesh" || meshDatabase.meshClass=="TriList" ) { + } else if ( meshDatabase.meshClass == "TriMesh" || meshDatabase.meshClass == "TriList" ) { Array coords; Array tri; silo::readTriMesh( fid, database.name, coords, tri ); - ASSERT( tri.size(1)==3 && coords.size(1)==3 ); - int N_tri = tri.size(0); - int N_point = coords.size(0); + ASSERT( tri.size( 1 ) == 3 && coords.size( 1 ) == 3 ); + int N_tri = tri.size( 0 ); + int N_point = coords.size( 0 ); std::shared_ptr mesh2( new IO::TriMesh( N_tri, N_point ) ); - for (int i=0; ivertices->points[i].x = coords(i,0); - mesh2->vertices->points[i].y = coords(i,1); - mesh2->vertices->points[i].z = coords(i,2); + for ( int i = 0; i < N_point; i++ ) { + mesh2->vertices->points[i].x = coords( i, 0 ); + mesh2->vertices->points[i].y = coords( i, 1 ); + mesh2->vertices->points[i].z = coords( i, 2 ); } - for (int i=0; iA[i] = tri(i,0); - mesh2->B[i] = tri(i,1); - mesh2->C[i] = tri(i,2); + for ( int i = 0; i < N_tri; i++ ) { + mesh2->A[i] = tri( i, 0 ); + mesh2->B[i] = tri( i, 1 ); + mesh2->C[i] = tri( i, 2 ); } - if ( meshDatabase.meshClass=="TriMesh" ) { + if ( meshDatabase.meshClass == "TriMesh" ) { mesh = mesh2; - } else if ( meshDatabase.meshClass=="TriList" ) { + } else if ( meshDatabase.meshClass == "TriList" ) { auto trilist = IO::getTriList( std::dynamic_pointer_cast( mesh2 ) ); - mesh = trilist; + mesh = trilist; } - } else if ( meshDatabase.meshClass=="DomainMesh" ) { + } else if ( meshDatabase.meshClass == "DomainMesh" ) { std::vector range; std::vector N; silo::readUniformMesh( fid, database.name, range, N ); - auto rankinfo = silo::read( fid, database.name+"_rankinfo" ); + auto rankinfo = silo::read( fid, database.name + "_rankinfo" ); RankInfoStruct rank_data( rankinfo[0], rankinfo[1], rankinfo[2], rankinfo[3] ); - mesh.reset( new IO::DomainMesh( rank_data, N[0], N[1], N[2], range[1]-range[0], range[3]-range[2], range[5]-range[4] ) ); + mesh.reset( new IO::DomainMesh( rank_data, N[0], N[1], N[2], range[1] - range[0], + range[3] - range[2], range[5] - range[4] ) ); } else { - ERROR("Unknown mesh class"); + ERROR( "Unknown mesh class" ); } silo::close( fid ); #else - ERROR("Build without silo support"); + ERROR( "Build without silo support" ); #endif } else { - ERROR("Unknown format"); + ERROR( "Unknown format" ); } - PROFILE_STOP("getMesh"); + PROFILE_STOP( "getMesh" ); return mesh; } // Read the given variable for the given mesh domain -std::shared_ptr IO::getVariable( const std::string& path, const std::string& timestep, - const MeshDatabase& meshDatabase, int domain, const std::string& variable ) +std::shared_ptr IO::getVariable( const std::string &path, const std::string ×tep, + const MeshDatabase &meshDatabase, int domain, const std::string &variable ) { - std::pair key(meshDatabase.domains[domain].name,variable); - std::map,DatabaseEntry>::const_iterator it; - it = meshDatabase.variable_data.find(key); - if ( it==meshDatabase.variable_data.end() ) + std::pair key( meshDatabase.domains[domain].name, variable ); + std::map, DatabaseEntry>::const_iterator it; + it = meshDatabase.variable_data.find( key ); + if ( it == meshDatabase.variable_data.end() ) return std::shared_ptr(); std::shared_ptr var; if ( meshDatabase.format == 2 ) { - const DatabaseEntry& database = it->second; - std::string filename = path + "/" + timestep + "/" + database.file; - FILE *fid = fopen(filename.c_str(),"rb"); - fseek(fid,database.offset,SEEK_SET); + const DatabaseEntry &database = it->second; + std::string filename = path + "/" + timestep + "/" + database.file; + FILE *fid = fopen( filename.c_str(), "rb" ); + fseek( fid, database.offset, SEEK_SET ); char line[1000]; - fgetl(line,1000,fid); - size_t i1 = find(line,':'); - size_t i2 = find(&line[i1+1],':')+i1+1; - std::vector values = splitList(&line[i2+1],','); - ASSERT(values.size()==5); - int dim = atoi(values[0].c_str()); - int type = atoi(values[1].c_str()); - size_t N = atol(values[2].c_str()); - size_t bytes = atol(values[3].c_str()); - std::string precision = values[4]; - var = std::shared_ptr( new IO::Variable() ); - var->dim = dim; - var->type = static_cast(type); - var->name = variable; - var->data.resize(N*dim); - if ( precision=="double" ) { - size_t count = fread(var->data.data(),sizeof(double),N*dim,fid); - ASSERT(count*sizeof(double)==bytes); + fgetl( line, 1000, fid ); + size_t i1 = find( line, ':' ); + size_t i2 = find( &line[i1 + 1], ':' ) + i1 + 1; + std::vector values = splitList( &line[i2 + 1], ',' ); + ASSERT( values.size() == 5 ); + int dim = atoi( values[0].c_str() ); + int type = atoi( values[1].c_str() ); + size_t N = atol( values[2].c_str() ); + size_t bytes = atol( values[3].c_str() ); + std::string precision = values[4]; + var = std::shared_ptr( new IO::Variable() ); + var->dim = dim; + var->type = static_cast( type ); + var->name = variable; + var->data.resize( N, dim ); + if ( precision == "double" ) { + size_t count = fread( var->data.data(), sizeof( double ), N * dim, fid ); + ASSERT( count * sizeof( double ) == bytes ); } else { - ERROR("Format not implimented"); + ERROR( "Format not implimented" ); } - fclose(fid); + fclose( fid ); } else if ( meshDatabase.format == 4 ) { // Reading a silo file #ifdef USE_SILO - const auto& database = meshDatabase.domains[domain]; + const auto &database = meshDatabase.domains[domain]; auto variableDatabase = meshDatabase.getVariableDatabase( variable ); - std::string filename = path + "/" + timestep + "/" + database.file; - auto fid = silo::open( filename, silo::READ ); + std::string filename = path + "/" + timestep + "/" + database.file; + auto fid = silo::open( filename, silo::READ ); var.reset( new Variable( variableDatabase.dim, variableDatabase.type, variable ) ); - if ( meshDatabase.meshClass=="PointList" ) { + if ( meshDatabase.meshClass == "PointList" ) { var->data = silo::readPointMeshVariable( fid, variable ); - } else if ( meshDatabase.meshClass=="TriMesh" || meshDatabase.meshClass=="TriList" ) { + } else if ( meshDatabase.meshClass == "TriMesh" || meshDatabase.meshClass == "TriList" ) { var->data = silo::readTriMeshVariable( fid, variable ); - } else if ( meshDatabase.meshClass=="DomainMesh" ) { + } else if ( meshDatabase.meshClass == "DomainMesh" ) { var->data = silo::readUniformMeshVariable( fid, variable ); } else { - ERROR("Unknown mesh class"); + ERROR( "Unknown mesh class" ); } silo::close( fid ); #else - ERROR("Build without silo support"); + ERROR( "Build without silo support" ); #endif } else { - ERROR("Unknown format"); + ERROR( "Unknown format" ); } return var; } /**************************************************** -* Reformat the variable to match the mesh * -****************************************************/ -void IO::reformatVariable( const IO::Mesh& mesh, IO::Variable& var ) + * Reformat the variable to match the mesh * + ****************************************************/ +void IO::reformatVariable( const IO::Mesh &mesh, IO::Variable &var ) { if ( mesh.className() == "DomainMesh" ) { - const IO::DomainMesh& mesh2 = dynamic_cast( mesh ); + const IO::DomainMesh &mesh2 = dynamic_cast( mesh ); if ( var.type == VariableType::NodeVariable ) { - size_t N2 = var.data.length() / ((mesh2.nx+1)*(mesh2.ny+1)*(mesh2.nz+1)); - ASSERT( (mesh2.nx+1)*(mesh2.ny+1)*(mesh2.nz+1)*N2 == var.data.length() ); - var.data.reshape( { (size_t) mesh2.nx+1, (size_t) mesh2.ny+1, (size_t) mesh2.nz+1, N2 } ); + size_t N2 = + var.data.length() / ( ( mesh2.nx + 1 ) * ( mesh2.ny + 1 ) * ( mesh2.nz + 1 ) ); + ASSERT( + ( mesh2.nx + 1 ) * ( mesh2.ny + 1 ) * ( mesh2.nz + 1 ) * N2 == var.data.length() ); + var.data.reshape( + { (size_t) mesh2.nx + 1, (size_t) mesh2.ny + 1, (size_t) mesh2.nz + 1, N2 } ); } else if ( var.type == VariableType::EdgeVariable ) { - ERROR("Not finished"); + ERROR( "Not finished" ); } else if ( var.type == VariableType::SurfaceVariable ) { - ERROR("Not finished"); + ERROR( "Not finished" ); } else if ( var.type == VariableType::VolumeVariable ) { - size_t N2 = var.data.length() / (mesh2.nx*mesh2.ny*mesh2.nz); - ASSERT( mesh2.nx*mesh2.ny*mesh2.nz*N2 == var.data.length() ); + size_t N2 = var.data.length() / ( mesh2.nx * mesh2.ny * mesh2.nz ); + ASSERT( mesh2.nx * mesh2.ny * mesh2.nz * N2 == var.data.length() ); var.data.reshape( { (size_t) mesh2.nx, (size_t) mesh2.ny, (size_t) mesh2.nz, N2 } ); } else { - ERROR("Invalid variable type"); + ERROR( "Invalid variable type" ); } } else if ( mesh.className() == "PointList" ) { - const IO::PointList& mesh2 = dynamic_cast( mesh ); - size_t N = mesh2.points.size(); - size_t N_var = var.data.length()/N; - ASSERT( N*N_var == var.data.length() ); + const IO::PointList &mesh2 = dynamic_cast( mesh ); + size_t N = mesh2.points.size(); + size_t N_var = var.data.length() / N; + ASSERT( N * N_var == var.data.length() ); var.data.reshape( { N, N_var } ); - } else if ( mesh.className()=="TriMesh" || mesh.className() == "TriList" ) { - std::shared_ptr mesh_ptr( const_cast(&mesh), []( void* ) {} ); + } else if ( mesh.className() == "TriMesh" || mesh.className() == "TriList" ) { + std::shared_ptr mesh_ptr( const_cast( &mesh ), []( void * ) {} ); std::shared_ptr mesh2 = getTriMesh( mesh_ptr ); if ( var.type == VariableType::NodeVariable ) { - size_t N = mesh2->vertices->points.size(); - size_t N_var = var.data.length()/N; - ASSERT( N*N_var == var.data.length() ); + size_t N = mesh2->vertices->points.size(); + size_t N_var = var.data.length() / N; + ASSERT( N * N_var == var.data.length() ); var.data.reshape( { N, N_var } ); } else if ( var.type == VariableType::EdgeVariable ) { - ERROR("Not finished"); + ERROR( "Not finished" ); } else if ( var.type == VariableType::SurfaceVariable ) { - ERROR("Not finished"); + ERROR( "Not finished" ); } else if ( var.type == VariableType::VolumeVariable ) { - size_t N = mesh2->A.size(); - size_t N_var = var.data.length()/N; - ASSERT( N*N_var == var.data.length() ); + size_t N = mesh2->A.size(); + size_t N_var = var.data.length() / N; + ASSERT( N * N_var == var.data.length() ); var.data.reshape( { N, N_var } ); } else { - ERROR("Invalid variable type"); + ERROR( "Invalid variable type" ); } } else { - ERROR("Unknown mesh type"); + ERROR( "Unknown mesh type" ); } } - - - diff --git a/IO/Reader.h b/IO/Reader.h index 7cc96ab7..6542a2ea 100644 --- a/IO/Reader.h +++ b/IO/Reader.h @@ -14,7 +14,24 @@ namespace IO { //! Get the path to a file -std::string getPath( const std::string& filename ); +std::string getPath( const std::string &filename ); + + +/*! + * @brief Get the maximum number of domains written + * @details This function reads the summary files to determine the maximum + * number of domains in the output. + * @param[in] path The path to use for reading + * @param[in] format The data format to use: + * old - Old mesh format (provided for backward compatibility) + * new - New format, 1 file/process + * silo - Silo + * auto - Auto-determin the format + * @param[in] comm Optional comm to use to reduce IO load by + * reading on rank 0 and then communicating the result + */ +int maxDomains( const std::string &path, const std::string &format = "auto", + const Utilities::MPI &comm = MPI_COMM_SELF ); /*! @@ -22,31 +39,34 @@ std::string getPath( const std::string& filename ); * @details This function reads the timestep list from the summary file. * @param[in] path The path to use for reading * @param[in] format The data format to use: - * old - Old mesh format (provided for backward compatibility) - * new - New format, 1 file/process - * silo - Silo - * @return append Append any existing data (default is false) + * old - Old mesh format (provided for backward compatibility) + * new - New format, 1 file/process + * silo - Silo + * auto - Auto-determin the format + * @return append Append any existing data (default is false) */ -std::vector readTimesteps( const std::string& path, const std::string& format ); +std::vector readTimesteps( + const std::string &path, const std::string &format = "auto" ); /*! * @brief Read the data for the timestep * @details This function reads the mesh and variable data provided for the given timestep. - * Note: this function requires that the number of ranks of the comm match the number of ranks in the meshes * @param[in] path The path to use for reading * @param[in] timestep The timestep iteration + * @param[in] domain The desired domain to read */ -std::vector readData( const std::string& path, const std::string& timestep, const Utilities::MPI &comm = MPI_COMM_WORLD ); +std::vector readData( + const std::string &path, const std::string ×tep, int domain ); //! Read the list of mesh databases for the given timestep -std::vector getMeshList( const std::string& path, const std::string& timestep ); +std::vector getMeshList( const std::string &path, const std::string ×tep ); //! Read the given mesh domain -std::shared_ptr getMesh( const std::string& path, const std::string& timestep, - const MeshDatabase& meshDatabase, int domain ); +std::shared_ptr getMesh( const std::string &path, const std::string ×tep, + const MeshDatabase &meshDatabase, int domain ); /*! @@ -59,8 +79,8 @@ std::shared_ptr getMesh( const std::string& path, const std::string& t * @param[in] variable The variable name to read * @return Returns the variable data as a linear array */ -std::shared_ptr getVariable( const std::string& path, const std::string& timestep, - const MeshDatabase& meshDatabase, int domain, const std::string& variable ); +std::shared_ptr getVariable( const std::string &path, const std::string ×tep, + const MeshDatabase &meshDatabase, int domain, const std::string &variable ); /*! @@ -69,9 +89,9 @@ std::shared_ptr getVariable( const std::string& path, const std::s * @param[in] mesh The underlying mesh * @param[in/out] variable The variable name to read */ -void reformatVariable( const IO::Mesh& mesh, IO::Variable& var ); +void reformatVariable( const IO::Mesh &mesh, IO::Variable &var ); -} // IO namespace +} // namespace IO #endif diff --git a/IO/Writer.cpp b/IO/Writer.cpp index 7414d5a1..d3f9d991 100644 --- a/IO/Writer.cpp +++ b/IO/Writer.cpp @@ -1,28 +1,69 @@ #include "IO/Writer.h" -#include "IO/MeshDatabase.h" #include "IO/IOHelpers.h" +#include "IO/MeshDatabase.h" #include "IO/silo.h" #include "common/MPI.h" #include "common/Utilities.h" -#include #include -#include -#include #include +#include +#include +#include - -enum class Format { OLD, NEW, SILO, UNKNOWN }; - +enum class Format { OLD, NEW, SILO, UNKNOWN }; /**************************************************** -* Initialize the writer * -****************************************************/ + * Recursively create the subdirectory * + ****************************************************/ +static void recursiveMkdir( const std::string &path, mode_t mode ) +{ + // Iterate through the root directories until we create the desired path + for ( size_t pos = 0; pos < path.size(); ) { + // slide backwards in string until next slash found + pos++; + for ( ; pos < path.size(); pos++ ) { + if ( path[pos] == '/' || path[pos] == 92 ) + break; + } + // Create the temporary path + auto path2 = path.substr( 0, pos ); + // Check if the temporary path exists + struct stat status; + int result = stat( path2.data(), &status ); + if ( result == 0 ) { + // if there is a part of the path that already exists make sure it is really a directory + if ( !S_ISDIR( status.st_mode ) ) { + ERROR( + "Error in recursiveMkdir...\n" + " Cannot create directories in path = " + + path + + "\n because some intermediate item in path exists and is NOT a directory" ); + } + continue; + } + // Create the directory and test the result + result = mkdir( path2.data(), mode ); + if ( result != 0 ) { + // Maybe another rank created the directory, check + int result = stat( path2.data(), &status ); + if ( result != 0 && !S_ISDIR( status.st_mode ) ) + ERROR( "Error in Utilities::recursiveMkdir...\n" + " Cannot create directory = " + + path2 ); + } + } +} + + +/**************************************************** + * Initialize the writer * + ****************************************************/ static std::string global_IO_path; static Format global_IO_format = Format::UNKNOWN; -void IO::initialize( const std::string& path, const std::string& format, bool append ) +void IO::initialize( const std::string &path, const std::string &format, bool append ) { if ( path.empty() ) global_IO_path = "."; @@ -35,161 +76,170 @@ void IO::initialize( const std::string& path, const std::string& format, bool ap else if ( format == "silo" ) global_IO_format = Format::SILO; else - ERROR("Unknown format"); - int rank = Utilities::MPI(MPI_COMM_WORLD).getRank(); - if ( !append && rank==0 ) { - mkdir(path.c_str(),S_IRWXU|S_IRGRP); + ERROR( "Unknown format" ); + int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank(); + if ( !append && rank == 0 ) { + recursiveMkdir( path, S_IRWXU | S_IRGRP ); std::string filename; - if ( global_IO_format==Format::OLD || global_IO_format==Format::NEW ) + if ( global_IO_format == Format::OLD || global_IO_format == Format::NEW ) filename = global_IO_path + "/summary.LBM"; - else if ( global_IO_format==Format::SILO ) + else if ( global_IO_format == Format::SILO ) filename = global_IO_path + "/LBM.visit"; else - ERROR("Unknown format"); - auto fid = fopen(filename.c_str(),"wb"); - fclose(fid); + ERROR( "Unknown format" ); + auto fid = fopen( filename.c_str(), "wb" ); + fclose( fid ); } } // Write the mesh data in the original format -static std::vector writeMeshesOrigFormat( const std::vector& meshData, const std::string& path ) +static std::vector writeMeshesOrigFormat( + const std::vector &meshData, const std::string &path, int rank ) { - int rank = Utilities::MPI(MPI_COMM_WORLD).getRank(); std::vector meshes_written; - for (size_t i=0; i mesh = meshData[i].mesh; IO::MeshDatabase mesh_entry; - mesh_entry.name = meshData[i].meshName; - mesh_entry.type = meshType(*mesh); + mesh_entry.name = meshData[i].meshName; + mesh_entry.type = meshType( *mesh ); mesh_entry.meshClass = meshData[i].mesh->className(); - mesh_entry.format = 1; + mesh_entry.format = 1; IO::DatabaseEntry domain; - domain.name = domainname; - domain.file = filename; + domain.name = domainname; + domain.file = filename; domain.offset = 0; - mesh_entry.domains.push_back(domain); + mesh_entry.domains.push_back( domain ); if ( !meshData[i].vars.empty() ) { - printf("Warning: variables are not supported with this format (original)\n"); - //for (size_t j=0; jname ); } const std::string meshClass = mesh->className(); - if ( meshClass=="PointList" ) { + if ( meshClass == "PointList" ) { // List of points - std::shared_ptr pointlist = std::dynamic_pointer_cast(mesh); - const std::vector& P = pointlist->points; - for (size_t i=0; i pointlist = + std::dynamic_pointer_cast( mesh ); + const std::vector &P = pointlist->points; + for ( size_t i = 0; i < P.size(); i++ ) { double x[3]; - x[0] = P[i].x; x[1] = P[i].y; x[2] = P[i].z; - fwrite(x,sizeof(double),3,fid); + x[0] = P[i].x; + x[1] = P[i].y; + x[2] = P[i].z; + fwrite( x, sizeof( double ), 3, fid ); } - } else if ( meshClass=="TriList" || meshClass=="TriMesh" ) { + } else if ( meshClass == "TriList" || meshClass == "TriMesh" ) { // Triangle mesh - std::shared_ptr trilist = IO::getTriList(mesh); - const std::vector& A = trilist->A; - const std::vector& B = trilist->B; - const std::vector& C = trilist->C; - for (size_t i=0; i trilist = IO::getTriList( mesh ); + const std::vector &A = trilist->A; + const std::vector &B = trilist->B; + const std::vector &C = trilist->C; + for ( size_t i = 0; i < A.size(); i++ ) { double tri[9]; - tri[0] = A[i].x; tri[1] = A[i].y; tri[2] = A[i].z; - tri[3] = B[i].x; tri[4] = B[i].y; tri[5] = B[i].z; - tri[6] = C[i].x; tri[7] = C[i].y; tri[8] = C[i].z; - fwrite(tri,sizeof(double),9,fid); + tri[0] = A[i].x; + tri[1] = A[i].y; + tri[2] = A[i].z; + tri[3] = B[i].x; + tri[4] = B[i].y; + tri[5] = B[i].z; + tri[6] = C[i].x; + tri[7] = C[i].y; + tri[8] = C[i].z; + fwrite( tri, sizeof( double ), 9, fid ); } - } else if ( meshClass=="DomainMesh" ) { + } else if ( meshClass == "DomainMesh" ) { // This format was never supported with the old format } else { - ERROR("Unknown mesh"); + ERROR( "Unknown mesh" ); } - fclose(fid); + fclose( fid ); std::sort( mesh_entry.variables.begin(), mesh_entry.variables.end() ); - mesh_entry.variables.erase( std::unique( mesh_entry.variables.begin(), mesh_entry.variables.end() ), mesh_entry.variables.end() ); - meshes_written.push_back(mesh_entry); + mesh_entry.variables.erase( + std::unique( mesh_entry.variables.begin(), mesh_entry.variables.end() ), + mesh_entry.variables.end() ); + meshes_written.push_back( mesh_entry ); } return meshes_written; } // Create the database entry for the mesh data -static IO::MeshDatabase getDatabase( const std::string& filename, const IO::MeshDataStruct& mesh, int format ) +static IO::MeshDatabase getDatabase( + const std::string &filename, const IO::MeshDataStruct &mesh, int format, int rank ) { - int rank = Utilities::MPI(MPI_COMM_WORLD).getRank(); char domainname[100]; - sprintf(domainname,"%s_%05i",mesh.meshName.c_str(),rank); + sprintf( domainname, "%s_%05i", mesh.meshName.c_str(), rank ); // Create the MeshDatabase IO::MeshDatabase database; - database.name = mesh.meshName; - database.type = meshType(*(mesh.mesh)); + database.name = mesh.meshName; + database.type = meshType( *( mesh.mesh ) ); database.meshClass = mesh.mesh->className(); - database.format = format; + database.format = format; // Write the mesh IO::DatabaseEntry domain; - domain.name = domainname; - domain.file = filename; + domain.name = domainname; + domain.file = filename; domain.offset = -1; - database.domains.push_back(domain); + database.domains.push_back( domain ); // Write the variables - for (size_t i=0; iname; info.type = mesh.vars[i]->type; - info.dim = mesh.vars[i]->dim; - database.variables.push_back(info); + info.dim = mesh.vars[i]->dim; + database.variables.push_back( info ); // Add domain variable info IO::DatabaseEntry variable; - variable.name = mesh.vars[i]->name; - variable.file = filename; + variable.name = mesh.vars[i]->name; + variable.file = filename; variable.offset = -1; - std::pair key(domain.name,mesh.vars[i]->name); - database.variable_data.insert( - std::pair,IO::DatabaseEntry>(key,variable) ); + std::pair key( domain.name, mesh.vars[i]->name ); + database.variable_data.insert( + std::pair, IO::DatabaseEntry>( key, variable ) ); } return database; } // Write a mesh (and variables) to a file -static IO::MeshDatabase write_domain( FILE *fid, const std::string& filename, - const IO::MeshDataStruct& mesh, int format ) +static IO::MeshDatabase write_domain( + FILE *fid, const std::string &filename, const IO::MeshDataStruct &mesh, int format, int rank ) { const int level = 0; - int rank = Utilities::MPI(MPI_COMM_WORLD).getRank(); // Create the MeshDatabase - IO::MeshDatabase database = getDatabase( filename, mesh, format ); + IO::MeshDatabase database = getDatabase( filename, mesh, format, rank ); // Write the mesh - IO::DatabaseEntry& domain = database.domains[0]; - domain.offset = ftell(fid); - std::pair data = mesh.mesh->pack(level); - fprintf(fid,"Mesh: %s-%05i: %lu\n",mesh.meshName.c_str(),rank,data.first); - fwrite(data.second,1,data.first,fid); - fprintf(fid,"\n"); - delete [] (char*) data.second; + IO::DatabaseEntry &domain = database.domains[0]; + domain.offset = ftell( fid ); + std::pair data = mesh.mesh->pack( level ); + fprintf( fid, "Mesh: %s-%05i: %lu\n", mesh.meshName.c_str(), rank, data.first ); + fwrite( data.second, 1, data.first, fid ); + fprintf( fid, "\n" ); + delete[]( char * ) data.second; // Write the variables - for (size_t i=0; i key(domain.name,mesh.vars[i]->name); - IO::DatabaseEntry& variable = database.variable_data[key]; - variable.offset = ftell(fid); - int dim = mesh.vars[i]->dim; - int type = static_cast(mesh.vars[i]->type); - size_t N = mesh.vars[i]->data.length(); - if ( type == static_cast(IO::VariableType::NullVariable) ) { - ERROR("Variable type not set"); + for ( size_t i = 0; i < mesh.vars.size(); i++ ) { + std::pair key( domain.name, mesh.vars[i]->name ); + IO::DatabaseEntry &variable = database.variable_data[key]; + variable.offset = ftell( fid ); + int dim = mesh.vars[i]->dim; + int type = static_cast( mesh.vars[i]->type ); + size_t N = mesh.vars[i]->data.length(); + if ( type == static_cast( IO::VariableType::NullVariable ) ) { + ERROR( "Variable type not set" ); } - size_t N_mesh = mesh.mesh->numberPointsVar(mesh.vars[i]->type); - ASSERT(N==dim*N_mesh); - fprintf(fid,"Var: %s-%05i-%s: %i, %i, %lu, %lu, double\n", - database.name.c_str(), rank, variable.name.c_str(), - dim, type, N_mesh, N*sizeof(double) ); - fwrite(mesh.vars[i]->data.data(),sizeof(double),N,fid); - fprintf(fid,"\n"); + size_t N_mesh = mesh.mesh->numberPointsVar( mesh.vars[i]->type ); + ASSERT( N == dim * N_mesh ); + fprintf( fid, "Var: %s-%05i-%s: %i, %i, %lu, %lu, double\n", database.name.c_str(), rank, + variable.name.c_str(), dim, type, N_mesh, N * sizeof( double ) ); + fwrite( mesh.vars[i]->data.data(), sizeof( double ), N, fid ); + fprintf( fid, "\n" ); } return database; } @@ -198,11 +248,12 @@ static IO::MeshDatabase write_domain( FILE *fid, const std::string& filename, #ifdef USE_SILO // Write a PointList mesh (and variables) to a file template -static void writeSiloPointMesh( DBfile *fid, const IO::PointList& mesh, const std::string& meshname ) +static void writeSiloPointMesh( + DBfile *fid, const IO::PointList &mesh, const std::string &meshname ) { - const auto& points = mesh.getPoints(); - std::vector x(points.size()), y(points.size()), z(points.size()); - for (size_t i=0; i x( points.size() ), y( points.size() ), z( points.size() ); + for ( size_t i = 0; i < x.size(); i++ ) { x[i] = points[i].x; y[i] = points[i].y; z[i] = points[i].z; @@ -210,28 +261,29 @@ static void writeSiloPointMesh( DBfile *fid, const IO::PointList& mesh, const st const TYPE *coords[] = { x.data(), y.data(), z.data() }; silo::writePointMesh( fid, meshname, 3, points.size(), coords ); } -static void writeSiloPointList( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database ) +static void writeSiloPointList( + DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database ) { - const IO::PointList& mesh = dynamic_cast( *meshData.mesh ); + const IO::PointList &mesh = dynamic_cast( *meshData.mesh ); const std::string meshname = database.domains[0].name; if ( meshData.precision == IO::DataType::Double ) { writeSiloPointMesh( fid, mesh, meshname ); } else if ( meshData.precision == IO::DataType::Float ) { writeSiloPointMesh( fid, mesh, meshname ); } else { - ERROR("Unsupported format"); + ERROR( "Unsupported format" ); } - const auto& points = mesh.getPoints(); - std::vector x(points.size()), y(points.size()), z(points.size()); - for (size_t i=0; i x( points.size() ), y( points.size() ), z( points.size() ); + for ( size_t i = 0; i < x.size(); i++ ) { x[i] = points[i].x; y[i] = points[i].y; z[i] = points[i].z; } const double *coords[] = { x.data(), y.data(), z.data() }; silo::writePointMesh( fid, meshname, 3, points.size(), coords ); - for (size_t i=0; i -static void writeSiloTriMesh( DBfile *fid, const IO::TriMesh& mesh, const std::string& meshname ) +static void writeSiloTriMesh( DBfile *fid, const IO::TriMesh &mesh, const std::string &meshname ) { - const auto& points = mesh.vertices->getPoints(); - std::vector x(points.size()), y(points.size()), z(points.size()); - for (size_t i=0; igetPoints(); + std::vector x( points.size() ), y( points.size() ), z( points.size() ); + for ( size_t i = 0; i < x.size(); i++ ) { x[i] = points[i].x; y[i] = points[i].y; z[i] = points[i].z; } const TYPE *coords[] = { x.data(), y.data(), z.data() }; - const int *tri[] = { mesh.A.data(), mesh.B.data(), mesh.C.data() }; + const int *tri[] = { mesh.A.data(), mesh.B.data(), mesh.C.data() }; silo::writeTriMesh( fid, meshname, 3, 2, points.size(), coords, mesh.A.size(), tri ); } -static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct& meshData, - const IO::TriMesh& mesh, IO::MeshDatabase database ) +static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct &meshData, + const IO::TriMesh &mesh, IO::MeshDatabase database ) { const std::string meshname = database.domains[0].name; if ( meshData.precision == IO::DataType::Double ) { @@ -271,11 +323,11 @@ static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct& meshData, } else if ( meshData.precision == IO::DataType::Float ) { writeSiloTriMesh( fid, mesh, meshname ); } else { - ERROR("Unsupported format"); + ERROR( "Unsupported format" ); } - for (size_t i=0; i( var.type ); + for ( size_t i = 0; i < meshData.vars.size(); i++ ) { + const IO::Variable &var = *meshData.vars[i]; + auto type = static_cast( var.type ); if ( var.precision == IO::DataType::Double ) { silo::writeTriMeshVariable( fid, 3, meshname, var.name, var.data, type ); } else if ( var.precision == IO::DataType::Float ) { @@ -287,35 +339,40 @@ static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct& meshData, data2.copy( var.data ); silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, type ); } else { - ERROR("Unsupported format"); + ERROR( "Unsupported format" ); } } } -static void writeSiloTriMesh( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database ) +static void writeSiloTriMesh( + DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database ) { - const IO::TriMesh& mesh = dynamic_cast( *meshData.mesh ); + const IO::TriMesh &mesh = dynamic_cast( *meshData.mesh ); writeSiloTriMesh2( fid, meshData, mesh, database ); } -static void writeSiloTriList( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database ) +static void writeSiloTriList( + DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database ) { auto mesh = getTriMesh( meshData.mesh ); writeSiloTriMesh2( fid, meshData, *mesh, database ); } // Write a DomainMesh mesh (and variables) to a file -static void writeSiloDomainMesh( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database ) +static void writeSiloDomainMesh( + DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database ) { - const IO::DomainMesh& mesh = dynamic_cast( *meshData.mesh ); + const IO::DomainMesh &mesh = dynamic_cast( *meshData.mesh ); RankInfoStruct info( mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz ); - std::array range = { info.ix*mesh.Lx/info.nx, (info.ix+1)*mesh.Lx/info.nx, - info.jy*mesh.Ly/info.ny, (info.jy+1)*mesh.Ly/info.ny, - info.kz*mesh.Lz/info.nz, (info.kz+1)*mesh.Lz/info.nz }; - std::array N = { mesh.nx, mesh.ny, mesh.nz }; - auto meshname = database.domains[0].name; + std::array range = { info.ix * mesh.Lx / info.nx, + ( info.ix + 1 ) * mesh.Lx / info.nx, info.jy * mesh.Ly / info.ny, + ( info.jy + 1 ) * mesh.Ly / info.ny, info.kz * mesh.Lz / info.nz, + ( info.kz + 1 ) * mesh.Lz / info.nz }; + std::array N = { mesh.nx, mesh.ny, mesh.nz }; + auto meshname = database.domains[0].name; silo::writeUniformMesh<3>( fid, meshname, range, N ); - silo::write( fid, meshname+"_rankinfo", { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz } ); - for (size_t i=0; i( var.type ); + silo::write( + fid, meshname + "_rankinfo", { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz } ); + for ( size_t i = 0; i < meshData.vars.size(); i++ ) { + const auto &var = *meshData.vars[i]; + auto type = static_cast( var.type ); if ( var.precision == IO::DataType::Double ) { silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, var.data, type ); } else if ( var.precision == IO::DataType::Float ) { @@ -327,65 +384,66 @@ static void writeSiloDomainMesh( DBfile *fid, const IO::MeshDataStruct& meshData data2.copy( var.data ); silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, type ); } else { - ERROR("Unsupported format"); + ERROR( "Unsupported format" ); } } } // Write a mesh (and variables) to a file -static IO::MeshDatabase write_domain_silo( DBfile *fid, const std::string& filename, - const IO::MeshDataStruct& mesh, int format ) +static IO::MeshDatabase write_domain_silo( + DBfile *fid, const std::string &filename, const IO::MeshDataStruct &mesh, int format, int rank ) { // Create the MeshDatabase - auto database = getDatabase( filename, mesh, format ); - if ( database.meshClass=="PointList" ) { + auto database = getDatabase( filename, mesh, format, rank ); + if ( database.meshClass == "PointList" ) { writeSiloPointList( fid, mesh, database ); - } else if ( database.meshClass=="TriMesh" ) { + } else if ( database.meshClass == "TriMesh" ) { writeSiloTriMesh( fid, mesh, database ); - } else if ( database.meshClass=="TriList" ) { + } else if ( database.meshClass == "TriList" ) { writeSiloTriList( fid, mesh, database ); - } else if ( database.meshClass=="DomainMesh" ) { + } else if ( database.meshClass == "DomainMesh" ) { writeSiloDomainMesh( fid, mesh, database ); } else { - ERROR("Unknown mesh class"); + ERROR( "Unknown mesh class" ); } return database; } // Write the summary file for silo -std::pair getSiloMeshType( const std::string& meshClass ) +std::pair getSiloMeshType( const std::string &meshClass ) { int meshType = 0; - int varType = 0; - if ( meshClass=="PointList" ) { + int varType = 0; + if ( meshClass == "PointList" ) { meshType = DB_POINTMESH; varType = DB_POINTVAR; - } else if ( meshClass=="TriMesh" ) { + } else if ( meshClass == "TriMesh" ) { meshType = DB_UCDMESH; varType = DB_UCDVAR; - } else if ( meshClass=="TriList" ) { + } else if ( meshClass == "TriList" ) { meshType = DB_UCDMESH; varType = DB_UCDVAR; - } else if ( meshClass=="DomainMesh" ) { + } else if ( meshClass == "DomainMesh" ) { meshType = DB_QUAD_RECT; varType = DB_QUADVAR; } else { - ERROR("Unknown mesh class"); + ERROR( "Unknown mesh class" ); } return std::make_pair( meshType, varType ); } -void writeSiloSummary( const std::vector& meshes_written, const std::string& filename ) +void writeSiloSummary( + const std::vector &meshes_written, const std::string &filename ) { auto fid = silo::open( filename, silo::CREATE ); - for ( const auto& data : meshes_written ) { + for ( const auto &data : meshes_written ) { auto type = getSiloMeshType( data.meshClass ); std::vector meshTypes( data.domains.size(), type.first ); std::vector varTypes( data.domains.size(), type.second ); std::vector meshNames; - for ( const auto& tmp : data.domains ) + for ( const auto &tmp : data.domains ) meshNames.push_back( tmp.file + ":" + tmp.name ); silo::writeMultiMesh( fid, data.name, meshNames, meshTypes ); - for (const auto& variable : data.variables ) { + for ( const auto &variable : data.variables ) { std::vector varnames; - for ( const auto& tmp : data.domains ) + for ( const auto &tmp : data.domains ) varnames.push_back( tmp.file + ":" + variable.name ); silo::writeMultiVar( fid, variable.name, varnames, varTypes ); } @@ -396,113 +454,111 @@ void writeSiloSummary( const std::vector& meshes_written, cons // Write the mesh data in the new format -static std::vector writeMeshesNewFormat( - const std::vector& meshData, const std::string& path, int format ) +static std::vector writeMeshesNewFormat( + const std::vector &meshData, const std::string &path, int format, int rank ) { - int rank = Utilities::MPI(MPI_COMM_WORLD).getRank(); std::vector meshes_written; char filename[100], fullpath[200]; - sprintf(filename,"%05i",rank); - sprintf(fullpath,"%s/%s",path.c_str(),filename); - FILE *fid = fopen(fullpath,"wb"); - for (size_t i=0; i mesh = meshData[i].mesh; - meshes_written.push_back( write_domain(fid,filename,meshData[i],format) ); + meshes_written.push_back( write_domain( fid, filename, meshData[i], format, rank ) ); } - fclose(fid); + fclose( fid ); return meshes_written; } // Write the mesh data to silo -static std::vector writeMeshesSilo( - const std::vector& meshData, const std::string& path, int format ) +static std::vector writeMeshesSilo( + const std::vector &meshData, const std::string &path, int format, int rank ) { #ifdef USE_SILO - int rank = Utilities::MPI(MPI_COMM_WORLD).getRank(); std::vector meshes_written; char filename[100], fullpath[200]; - sprintf(filename,"%05i.silo",rank); - sprintf(fullpath,"%s/%s",path.c_str(),filename); + sprintf( filename, "%05i.silo", rank ); + sprintf( fullpath, "%s/%s", path.c_str(), filename ); auto fid = silo::open( fullpath, silo::CREATE ); - for (size_t i=0; i(); #endif -} +} /**************************************************** -* Write the mesh data * -****************************************************/ -void IO::writeData( const std::string& subdir, const std::vector& meshData, const Utilities::MPI& comm ) + * Write the mesh data * + ****************************************************/ +void IO::writeData( const std::string &subdir, const std::vector &meshData, + const Utilities::MPI &comm ) { if ( global_IO_path.empty() ) - IO::initialize( ); - PROFILE_START("writeData"); - int rank = Utilities::MPI(MPI_COMM_WORLD).getRank(); + IO::initialize(); + PROFILE_START( "writeData" ); + int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank(); // Check the meshData before writing - for ( const auto& data : meshData ) { + for ( const auto &data : meshData ) { if ( !data.check() ) - ERROR("Error in meshData"); + ERROR( "Error in meshData" ); } // Create the output directory std::string path = global_IO_path + "/" + subdir; - if ( rank == 0 ) { - mkdir(path.c_str(),S_IRWXU|S_IRGRP); - } - comm.barrier(); + recursiveMkdir( path, S_IRWXU | S_IRGRP ); // Write the mesh files std::vector meshes_written; if ( global_IO_format == Format::OLD ) { // Write the original triangle format - meshes_written = writeMeshesOrigFormat( meshData, path ); + meshes_written = writeMeshesOrigFormat( meshData, path, rank ); } else if ( global_IO_format == Format::NEW ) { // Write the new format (double precision) - meshes_written = writeMeshesNewFormat( meshData, path, 2 ); + meshes_written = writeMeshesNewFormat( meshData, path, 2, rank ); } else if ( global_IO_format == Format::SILO ) { // Write silo - meshes_written = writeMeshesSilo( meshData, path, 4 ); + meshes_written = writeMeshesSilo( meshData, path, 4, rank ); } else { - ERROR("Unknown format"); + ERROR( "Unknown format" ); } // Gather a complete list of files on rank 0 - meshes_written = gatherAll(meshes_written,comm); + meshes_written = gatherAll( meshes_written, comm ); // Write the summary files if ( rank == 0 ) { // Write the summary file for the current timestep char filename[200]; - sprintf(filename,"%s/LBM.summary",path.c_str()); - write(meshes_written,filename); - // Write summary silo file if needed - #ifdef USE_SILO + sprintf( filename, "%s/LBM.summary", path.c_str() ); + write( meshes_written, filename ); +// Write summary silo file if needed +#ifdef USE_SILO if ( global_IO_format == Format::SILO ) { - sprintf(filename,"%s/summary.silo",path.c_str()); - writeSiloSummary(meshes_written,filename); + sprintf( filename, "%s/summary.silo", path.c_str() ); + writeSiloSummary( meshes_written, filename ); } - #endif +#endif // Add the timestep to the global summary file if ( global_IO_format == Format::OLD || global_IO_format == Format::NEW ) { - auto filename = global_IO_path+"/summary.LBM"; - FILE *fid = fopen(filename.c_str(),"ab"); - fprintf(fid,"%s/\n",subdir.c_str()); - fclose(fid); + auto filename = global_IO_path + "/summary.LBM"; + FILE *fid = fopen( filename.c_str(), "ab" ); + fprintf( fid, "%s/\n", subdir.c_str() ); + fclose( fid ); } else if ( global_IO_format == Format::SILO ) { - auto filename = global_IO_path+"/LBM.visit"; - FILE *fid = fopen(filename.c_str(),"ab"); - fprintf(fid,"%s/summary.silo\n",subdir.c_str()); - fclose(fid); + auto filename = global_IO_path + "/LBM.visit"; + FILE *fid = fopen( filename.c_str(), "ab" ); + fprintf( fid, "%s/summary.silo\n", subdir.c_str() ); + fclose( fid ); } else { - ERROR("Unknown format"); + ERROR( "Unknown format" ); } } - PROFILE_STOP("writeData"); + PROFILE_STOP( "writeData" ); } - - diff --git a/IO/Writer.h b/IO/Writer.h index dfc22db8..c3d9d5bb 100644 --- a/IO/Writer.h +++ b/IO/Writer.h @@ -14,17 +14,18 @@ namespace IO { /*! * @brief Initialize the writer - * @details This function initializes the writer to the given path. All subsequent - * writes will occur in this directory. If this is not called, then it will default - * to the current path. + * @details This function initializes the writer to the given path. + * All subsequent writes will occur in this directory. + * If this is not called, then it will default to the current path. * @param[in] path The path to use for writes * @param[in] format The data format to use: - * old - Old mesh format (provided for backward compatibility, cannot write variables) - * new - New format, 1 file/process - * silo - Silo + * old - Old mesh format + * (provided for backward compatibility, cannot write variables) + * new - New format, 1 file/process silo - Silo * @param[in] append Append any existing data (default is false) */ -void initialize( const std::string& path="", const std::string& format="silo", bool append=false ); +void initialize( + const std::string &path = "", const std::string &format = "silo", bool append = false ); /*! @@ -34,7 +35,8 @@ void initialize( const std::string& path="", const std::string& format="silo", b * @param[in] meshData The data to write * @param[in] comm The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof) */ -void writeData( const std::string& subdir, const std::vector& meshData, const Utilities::MPI& comm ); +void writeData( const std::string &subdir, const std::vector &meshData, + const Utilities::MPI &comm ); /*! @@ -44,14 +46,15 @@ void writeData( const std::string& subdir, const std::vector * @param[in] meshData The data to write * @param[in] comm The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof) */ -inline void writeData( int timestep, const std::vector& meshData, const Utilities::MPI& comm ) +inline void writeData( + int timestep, const std::vector &meshData, const Utilities::MPI &comm ) { char subdir[100]; - sprintf(subdir,"vis%03i",timestep); + sprintf( subdir, "vis%03i", timestep ); writeData( subdir, meshData, comm ); } -} // IO namespace +} // namespace IO #endif diff --git a/IO/netcdf.cpp b/IO/netcdf.cpp index 6c3773e3..06f41dba 100644 --- a/IO/netcdf.cpp +++ b/IO/netcdf.cpp @@ -1,6 +1,6 @@ #include "IO/netcdf.h" -#include "common/Utilities.h" #include "common/MPI.h" +#include "common/Utilities.h" #include "ProfilerApp.h" @@ -12,14 +12,14 @@ #include -#define CHECK_NC_ERR( ERR ) \ - do { \ - if ( ERR != NC_NOERR ) { \ +#define CHECK_NC_ERR( ERR ) \ + do { \ + if ( ERR != NC_NOERR ) { \ std::string msg = "Error calling netcdf routine: "; \ - msg += nc_strerror( ERR ); \ - ERROR( msg ); \ - } \ - } while (0) + msg += nc_strerror( ERR ); \ + ERROR( msg ); \ + } \ + } while ( 0 ) namespace netcdf { @@ -50,43 +50,64 @@ static inline VariableType convertType( nc_type type ) else if ( type == NC_DOUBLE ) type2 = DOUBLE; else - ERROR("Unknown type"); + ERROR( "Unknown type" ); return type2; } // Get nc_type from the template -template inline nc_type getType(); -template<> inline nc_type getType() { return NC_CHAR; } -template<> inline nc_type getType() { return NC_SHORT; } -template<> inline nc_type getType() { return NC_INT; } -template<> inline nc_type getType() { return NC_FLOAT; } -template<> inline nc_type getType() { return NC_DOUBLE; } +template +inline nc_type getType(); +template<> +inline nc_type getType() +{ + return NC_CHAR; +} +template<> +inline nc_type getType() +{ + return NC_SHORT; +} +template<> +inline nc_type getType() +{ + return NC_INT; +} +template<> +inline nc_type getType() +{ + return NC_FLOAT; +} +template<> +inline nc_type getType() +{ + return NC_DOUBLE; +} // Function to reverse an array template -inline std::vector reverse( const std::vector& x ) +inline std::vector reverse( const std::vector &x ) { - std::vector y(x.size()); - for (size_t i=0; i y( x.size() ); + for ( size_t i = 0; i < x.size(); i++ ) + y[i] = x[x.size() - i - 1]; return y; } // Function to reverse an array template -inline std::vector convert( const std::vector& x ) +inline std::vector convert( const std::vector &x ) { - std::vector y(x.size()); - for (size_t i=0; i(x[i]); + std::vector y( x.size() ); + for ( size_t i = 0; i < x.size(); i++ ) + y[i] = static_cast( x[i] ); return y; } /**************************************************** -* Convert the VariableType to a string * -****************************************************/ + * Convert the VariableType to a string * + ****************************************************/ std::string VariableTypeName( VariableType type ) { if ( type == BYTE ) @@ -114,9 +135,9 @@ std::string VariableTypeName( VariableType type ) /**************************************************** -* Open/close a file * -****************************************************/ -int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm ) + * Open/close a file * + ****************************************************/ +int open( const std::string &filename, FileMode mode, const Utilities::MPI &comm ) { int fid = 0; if ( comm.isNull() ) { @@ -127,23 +148,26 @@ int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm int err = nc_open( filename.c_str(), NC_WRITE, &fid ); CHECK_NC_ERR( err ); } else if ( mode == CREATE ) { - int err = nc_create( filename.c_str(), NC_SHARE|NC_64BIT_OFFSET, &fid ); + int err = nc_create( filename.c_str(), NC_SHARE | NC_64BIT_OFFSET, &fid ); CHECK_NC_ERR( err ); } else { - ERROR("Unknown file mode"); + ERROR( "Unknown file mode" ); } } else { if ( mode == READ ) { - int err = nc_open_par( filename.c_str(), NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid ); + int err = nc_open_par( + filename.c_str(), NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid ); CHECK_NC_ERR( err ); } else if ( mode == WRITE ) { - int err = nc_open_par( filename.c_str(), NC_WRITE|NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid ); + int err = nc_open_par( filename.c_str(), NC_WRITE | NC_MPIPOSIX, comm.getCommunicator(), + MPI_INFO_NULL, &fid ); CHECK_NC_ERR( err ); } else if ( mode == CREATE ) { - int err = nc_create_par( filename.c_str(), NC_NETCDF4|NC_MPIIO, comm.getCommunicator(), MPI_INFO_NULL, &fid ); + int err = nc_create_par( filename.c_str(), NC_NETCDF4 | NC_MPIIO, + comm.getCommunicator(), MPI_INFO_NULL, &fid ); CHECK_NC_ERR( err ); } else { - ERROR("Unknown file mode"); + ERROR( "Unknown file mode" ); } } return fid; @@ -152,42 +176,42 @@ void close( int fid ) { int err = nc_close( fid ); if ( err != NC_NOERR ) - ERROR("Error closing file"); + ERROR( "Error closing file" ); } /**************************************************** -* Query basic properties * -****************************************************/ + * Query basic properties * + ****************************************************/ static std::vector getDimVar( int fid, int varid ) { int ndim = 0; - int err = nc_inq_varndims( fid, varid, &ndim ); + int err = nc_inq_varndims( fid, varid, &ndim ); CHECK_NC_ERR( err ); - std::vector dims(ndim,0); - int dimid[64] = {-1}; - err = nc_inq_vardimid( fid, varid, dimid ); + std::vector dims( ndim, 0 ); + int dimid[64] = { -1 }; + err = nc_inq_vardimid( fid, varid, dimid ); CHECK_NC_ERR( err ); - for (int i=0; i getVarDim( int fid, const std::string& var ) +std::vector getVarDim( int fid, const std::string &var ) { return getDimVar( fid, getVarID( fid, var ) ); } -std::vector getAttDim( int fid, const std::string& att ) +std::vector getAttDim( int fid, const std::string &att ) { - std::vector dim(1,0); + std::vector dim( 1, 0 ); int err = nc_inq_attlen( fid, NC_GLOBAL, att.c_str(), dim.data() ); CHECK_NC_ERR( err ); return dim; @@ -197,9 +221,9 @@ std::vector getVarNames( int fid ) int nvar; int err = nc_inq( fid, NULL, &nvar, NULL, NULL ); CHECK_NC_ERR( err ); - std::vector vars(nvar); - for (int i=0; i vars( nvar ); + for ( int i = 0; i < nvar; i++ ) { + char name[NC_MAX_NAME + 1]; err = nc_inq_varname( fid, i, name ); CHECK_NC_ERR( err ); vars[i] = name; @@ -211,262 +235,269 @@ std::vector getAttNames( int fid ) int natt; int err = nc_inq( fid, NULL, NULL, &natt, NULL ); CHECK_NC_ERR( err ); - std::vector att(natt); - for (int i=0; i att( natt ); + for ( int i = 0; i < natt; i++ ) { + char name[NC_MAX_NAME + 1]; + err = nc_inq_attname( fid, NC_GLOBAL, i, name ); CHECK_NC_ERR( err ); att[i] = name; } return att; } -VariableType getVarType( int fid, const std::string& var ) +VariableType getVarType( int fid, const std::string &var ) { int varid = -1; - int err = nc_inq_varid( fid, var.c_str(), &varid ); + int err = nc_inq_varid( fid, var.c_str(), &varid ); CHECK_NC_ERR( err ); - nc_type type=0; - err = nc_inq_vartype( fid, varid, &type ); + nc_type type = 0; + err = nc_inq_vartype( fid, varid, &type ); CHECK_NC_ERR( err ); - return convertType(type); + return convertType( type ); } -VariableType getAttType( int fid, const std::string& att ) +VariableType getAttType( int fid, const std::string &att ) { - nc_type type=0; - int err = nc_inq_atttype( fid, NC_GLOBAL, att.c_str(), &type ); + nc_type type = 0; + int err = nc_inq_atttype( fid, NC_GLOBAL, att.c_str(), &type ); CHECK_NC_ERR( err ); - return convertType(type); + return convertType( type ); } - /**************************************************** -* Read a variable * -****************************************************/ + * Read a variable * + ****************************************************/ template<> -Array getVar( int fid, const std::string& var ) +Array getVar( int fid, const std::string &var ) { - PROFILE_START("getVar"); - Array x( reverse(getVarDim(fid,var)) ); - int err = nc_get_var_ushort( fid, getVarID(fid,var), x.data() ); + PROFILE_START( "getVar" ); + Array x( reverse( getVarDim( fid, var ) ) ); + int err = nc_get_var_ushort( fid, getVarID( fid, var ), x.data() ); CHECK_NC_ERR( err ); - PROFILE_STOP("getVar"); + PROFILE_STOP( "getVar" ); return x.reverseDim(); } template<> -Array getVar( int fid, const std::string& var ) +Array getVar( int fid, const std::string &var ) { - PROFILE_START("getVar"); - Array x( reverse(getVarDim(fid,var)) ); - int err = nc_get_var_short( fid, getVarID(fid,var), x.data() ); + PROFILE_START( "getVar" ); + Array x( reverse( getVarDim( fid, var ) ) ); + int err = nc_get_var_short( fid, getVarID( fid, var ), x.data() ); CHECK_NC_ERR( err ); - PROFILE_STOP("getVar"); + PROFILE_STOP( "getVar" ); return x.reverseDim(); } template<> -Array getVar( int fid, const std::string& var ) +Array getVar( int fid, const std::string &var ) { - PROFILE_START("getVar"); - Array x( reverse(getVarDim(fid,var)) ); - int err = nc_get_var_uint( fid, getVarID(fid,var), x.data() ); + PROFILE_START( "getVar" ); + Array x( reverse( getVarDim( fid, var ) ) ); + int err = nc_get_var_uint( fid, getVarID( fid, var ), x.data() ); CHECK_NC_ERR( err ); - PROFILE_STOP("getVar"); + PROFILE_STOP( "getVar" ); return x.reverseDim(); } template<> -Array getVar( int fid, const std::string& var ) +Array getVar( int fid, const std::string &var ) { - PROFILE_START("getVar"); - Array x( reverse(getVarDim(fid,var)) ); - int err = nc_get_var_int( fid, getVarID(fid,var), x.data() ); + PROFILE_START( "getVar" ); + Array x( reverse( getVarDim( fid, var ) ) ); + int err = nc_get_var_int( fid, getVarID( fid, var ), x.data() ); CHECK_NC_ERR( err ); - PROFILE_STOP("getVar"); + PROFILE_STOP( "getVar" ); return x.reverseDim(); } template<> -Array getVar( int fid, const std::string& var ) +Array getVar( int fid, const std::string &var ) { - PROFILE_START("getVar"); - Array x( reverse(getVarDim(fid,var)) ); - int err = nc_get_var_float( fid, getVarID(fid,var), x.data() ); + PROFILE_START( "getVar" ); + Array x( reverse( getVarDim( fid, var ) ) ); + int err = nc_get_var_float( fid, getVarID( fid, var ), x.data() ); CHECK_NC_ERR( err ); - PROFILE_STOP("getVar"); + PROFILE_STOP( "getVar" ); return x.reverseDim(); } template<> -Array getVar( int fid, const std::string& var ) +Array getVar( int fid, const std::string &var ) { - PROFILE_START("getVar"); - Array x( reverse(getVarDim(fid,var)) ); - int err = nc_get_var_double( fid, getVarID(fid,var), x.data() ); + PROFILE_START( "getVar" ); + Array x( reverse( getVarDim( fid, var ) ) ); + int err = nc_get_var_double( fid, getVarID( fid, var ), x.data() ); CHECK_NC_ERR( err ); - PROFILE_STOP("getVar"); + PROFILE_STOP( "getVar" ); return x.reverseDim(); } template<> -Array getVar( int fid, const std::string& var ) -{ - PROFILE_START("getVar"); - Array x( reverse(getVarDim(fid,var)) ); - int err = nc_get_var_text( fid, getVarID(fid,var), x.data() ); +Array getVar( int fid, const std::string &var ) +{ + PROFILE_START( "getVar" ); + Array x( reverse( getVarDim( fid, var ) ) ); + int err = nc_get_var_text( fid, getVarID( fid, var ), x.data() ); CHECK_NC_ERR( err ); - PROFILE_STOP("getVar"); + PROFILE_STOP( "getVar" ); return x.reverseDim(); } template<> -Array getVar( int fid, const std::string& var ) +Array getVar( int fid, const std::string &var ) { - PROFILE_START("getVar"); - Array tmp = getVar( fid, var ); - std::vector dim = {tmp.size(0), tmp.size(1), tmp.size(2) }; + PROFILE_START( "getVar" ); + Array tmp = getVar( fid, var ); + std::vector dim = { tmp.size( 0 ), tmp.size( 1 ), tmp.size( 2 ) }; if ( dim.size() == 1 ) dim[0] = 1; else dim.erase( dim.begin() ); - Array text(dim); - for (size_t i=0; i"); + Array text( dim ); + for ( size_t i = 0; i < text.length(); i++ ) + text( i ) = &( tmp( 0, i ) ); + PROFILE_STOP( "getVar" ); return text; } -static inline void get_stride_args( const std::vector& start, - const std::vector& count, const std::vector& stride, - size_t *startp, size_t *countp, ptrdiff_t *stridep ) +static inline void get_stride_args( const std::vector &start, const std::vector &count, + const std::vector &stride, size_t *startp, size_t *countp, ptrdiff_t *stridep ) { - for (size_t i=0; i -int nc_get_vars_TYPE( int fid, int varid, const size_t start[], - const size_t count[], const ptrdiff_t stride[], TYPE *ptr ); +int nc_get_vars_TYPE( int fid, int varid, const size_t start[], const size_t count[], + const ptrdiff_t stride[], TYPE *ptr ); template<> -int nc_get_vars_TYPE( int fid, int varid, const size_t start[], - const size_t count[], const ptrdiff_t stride[], short *ptr ) +int nc_get_vars_TYPE( int fid, int varid, const size_t start[], const size_t count[], + const ptrdiff_t stride[], short *ptr ) { return nc_get_vars_short( fid, varid, start, count, stride, ptr ); } template<> -int nc_get_vars_TYPE( int fid, int varid, const size_t start[], - const size_t count[], const ptrdiff_t stride[], int *ptr ) +int nc_get_vars_TYPE( int fid, int varid, const size_t start[], const size_t count[], + const ptrdiff_t stride[], int *ptr ) { return nc_get_vars_int( fid, varid, start, count, stride, ptr ); } template<> -int nc_get_vars_TYPE( int fid, int varid, const size_t start[], - const size_t count[], const ptrdiff_t stride[], float *ptr ) +int nc_get_vars_TYPE( int fid, int varid, const size_t start[], const size_t count[], + const ptrdiff_t stride[], float *ptr ) { return nc_get_vars_float( fid, varid, start, count, stride, ptr ); } template<> -int nc_get_vars_TYPE( int fid, int varid, const size_t start[], - const size_t count[], const ptrdiff_t stride[], double *ptr ) +int nc_get_vars_TYPE( int fid, int varid, const size_t start[], const size_t count[], + const ptrdiff_t stride[], double *ptr ) { return nc_get_vars_double( fid, varid, start, count, stride, ptr ); } template -Array getVar( int fid, const std::string& var, const std::vector& start, - const std::vector& count, const std::vector& stride ) +Array getVar( int fid, const std::string &var, const std::vector &start, + const std::vector &count, const std::vector &stride ) { - PROFILE_START("getVar<> (strided)"); + PROFILE_START( "getVar<> (strided)" ); std::vector var_size = getVarDim( fid, var ); - for (int d=0; d<(int)var_size.size(); d++) { - if ( start[d]<0 || start[d]+stride[d]*(count[d]-1)>(int)var_size[d] ) { - int rank = Utilities::MPI(MPI_COMM_WORLD).getRank(); + for ( int d = 0; d < (int) var_size.size(); d++ ) { + if ( start[d] < 0 || start[d] + stride[d] * ( count[d] - 1 ) > (int) var_size[d] ) { + int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank(); char tmp[1000]; - sprintf(tmp,"%i: Range exceeded array dimension:\n" + sprintf( tmp, + "%i: Range exceeded array dimension:\n" " start[%i]=%i, count[%i]=%i, stride[%i]=%i, var_size[%i]=%i", - rank,d,start[d],d,count[d],d,stride[d],d,(int)var_size[d]); - ERROR(tmp); + rank, d, start[d], d, count[d], d, stride[d], d, (int) var_size[d] ); + ERROR( tmp ); } } - Array x( reverse(convert(count)) ); + Array x( reverse( convert( count ) ) ); size_t startp[10], countp[10]; ptrdiff_t stridep[10]; get_stride_args( start, count, stride, startp, countp, stridep ); - int err = nc_get_vars_TYPE( fid, getVarID(fid,var), startp, countp, stridep, x.data() ); + int err = + nc_get_vars_TYPE( fid, getVarID( fid, var ), startp, countp, stridep, x.data() ); CHECK_NC_ERR( err ); - PROFILE_STOP("getVar<> (strided)"); + PROFILE_STOP( "getVar<> (strided)" ); return x.reverseDim(); } -template Array getVar( int, const std::string&, const std::vector&, const std::vector&, const std::vector& ); -template Array getVar( int, const std::string&, const std::vector&, const std::vector&, const std::vector& ); -template Array getVar( int, const std::string&, const std::vector&, const std::vector&, const std::vector& ); -template Array getVar( int, const std::string&, const std::vector&, const std::vector&, const std::vector& ); +template Array getVar( int, const std::string &, const std::vector &, + const std::vector &, const std::vector & ); +template Array getVar( int, const std::string &, const std::vector &, + const std::vector &, const std::vector & ); +template Array getVar( int, const std::string &, const std::vector &, + const std::vector &, const std::vector & ); +template Array getVar( int, const std::string &, const std::vector &, + const std::vector &, const std::vector & ); /**************************************************** -* Read an attribute * -****************************************************/ + * Read an attribute * + ****************************************************/ template<> -Array getAtt( int fid, const std::string& att ) +Array getAtt( int fid, const std::string &att ) { - PROFILE_START("getAtt"); - Array x( getAttDim(fid,att) ); + PROFILE_START( "getAtt" ); + Array x( getAttDim( fid, att ) ); int err = nc_get_att_double( fid, NC_GLOBAL, att.c_str(), x.data() ); CHECK_NC_ERR( err ); - PROFILE_STOP("getAtt"); + PROFILE_STOP( "getAtt" ); return x; } template<> -Array getAtt( int fid, const std::string& att ) +Array getAtt( int fid, const std::string &att ) { - PROFILE_START("getAtt"); - char *tmp = new char[getAttDim(fid,att)[0]]; - Array x(1); - x(0) = tmp; - delete [] tmp; - PROFILE_STOP("getAtt"); + PROFILE_START( "getAtt" ); + char *tmp = new char[getAttDim( fid, att )[0]]; + Array x( 1 ); + x( 0 ) = tmp; + delete[] tmp; + PROFILE_STOP( "getAtt" ); return x; } /**************************************************** -* Write an array to a file * -****************************************************/ -std::vector defDim( int fid, const std::vector& names, const std::vector& dims ) + * Write an array to a file * + ****************************************************/ +std::vector defDim( + int fid, const std::vector &names, const std::vector &dims ) { - std::vector dimid(names.size(),0); - for (size_t i=0; i dimid( names.size(), 0 ); + for ( size_t i = 0; i < names.size(); i++ ) { + int err = nc_def_dim( fid, names[i].c_str(), dims[i], &dimid[i] ); CHECK_NC_ERR( err ); } return dimid; } template -void write( int fid, const std::string& var, const std::vector& dimids, - const Array& data, const RankInfoStruct& info ) +void write( int fid, const std::string &var, const std::vector &dimids, + const Array &data, const RankInfoStruct &info ) { // Define the variable int varid = 0; - int err = nc_def_var( fid, var.c_str(), getType(), data.ndim(), dimids.data(), &varid ); + int err = nc_def_var( fid, var.c_str(), getType(), data.ndim(), dimids.data(), &varid ); CHECK_NC_ERR( err ); - // exit define mode + // exit define mode err = nc_enddef( fid ); CHECK_NC_ERR( err ); - // set the access method to use MPI/PnetCDF collective I/O + // set the access method to use MPI/PnetCDF collective I/O err = nc_var_par_access( fid, varid, NC_INDEPENDENT ); CHECK_NC_ERR( err ); // parallel write: each process writes its subarray to the file - auto x = data.reverseDim(); - std::vector count = { data.size(0), data.size(1), data.size(2) }; - std::vector start = { info.ix*data.size(0), info.jy*data.size(1), info.kz*data.size(2) }; + auto x = data.reverseDim(); + std::vector count = { data.size( 0 ), data.size( 1 ), data.size( 2 ) }; + std::vector start = { info.ix * data.size( 0 ), info.jy * data.size( 1 ), + info.kz * data.size( 2 ) }; nc_put_vara( fid, varid, start.data(), count.data(), x.data() ); } -template void write( int fid, const std::string& var, const std::vector& dimids, const Array& data, const RankInfoStruct& info ); -template void write( int fid, const std::string& var, const std::vector& dimids, const Array& data, const RankInfoStruct& info ); -template void write( int fid, const std::string& var, const std::vector& dimids, const Array& data, const RankInfoStruct& info ); -template void write( int fid, const std::string& var, const std::vector& dimids, const Array& data, const RankInfoStruct& info ); +template void write( int fid, const std::string &var, const std::vector &dimids, + const Array &data, const RankInfoStruct &info ); +template void write( int fid, const std::string &var, const std::vector &dimids, + const Array &data, const RankInfoStruct &info ); +template void write( int fid, const std::string &var, const std::vector &dimids, + const Array &data, const RankInfoStruct &info ); +template void write( int fid, const std::string &var, const std::vector &dimids, + const Array &data, const RankInfoStruct &info ); - -}; // netcdf namespace +}; // namespace netcdf #else #endif - - diff --git a/IO/netcdf.h b/IO/netcdf.h index e1f65e61..eb77784d 100644 --- a/IO/netcdf.h +++ b/IO/netcdf.h @@ -5,9 +5,8 @@ #include #include "common/Array.h" -#include "common/MPI.h" #include "common/Communication.h" - +#include "common/MPI.h" namespace netcdf { @@ -31,15 +30,15 @@ std::string VariableTypeName( VariableType type ); * @param filename File to open * @param mode Open the file for reading or writing * @param comm MPI communicator to use (MPI_COMM_WORLD: don't use parallel netcdf) -*/ -int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm=MPI_COMM_NULL ); + */ +int open( const std::string &filename, FileMode mode, const Utilities::MPI &comm = MPI_COMM_NULL ); /*! * @brief Close netcdf file * @details This function closes a netcdf file * @param fid Handle to the open file -*/ + */ void close( int fid ); @@ -47,7 +46,7 @@ void close( int fid ); * @brief Read the variable names * @details This function reads a list of the variable names in the file * @param fid Handle to the open file -*/ + */ std::vector getVarNames( int fid ); @@ -55,7 +54,7 @@ std::vector getVarNames( int fid ); * @brief Read the attribute names * @details This function reads a list of the attribute names in the file * @param fid Handle to the open file -*/ + */ std::vector getAttNames( int fid ); @@ -64,8 +63,8 @@ std::vector getAttNames( int fid ); * @details This function returns the type for a variable * @param fid Handle to the open file * @param var Variable to read -*/ -VariableType getVarType( int fid, const std::string& var ); + */ +VariableType getVarType( int fid, const std::string &var ); /*! @@ -73,8 +72,8 @@ VariableType getVarType( int fid, const std::string& var ); * @details This function returns the type for an attribute * @param fid Handle to the open file * @param att Attribute to read -*/ -VariableType getAttType( int fid, const std::string& att ); + */ +VariableType getAttType( int fid, const std::string &att ); /*! @@ -82,8 +81,8 @@ VariableType getAttType( int fid, const std::string& att ); * @details This function returns the die for a variable * @param fid Handle to the open file * @param var Variable to read -*/ -std::vector getVarDim( int fid, const std::string& var ); + */ +std::vector getVarDim( int fid, const std::string &var ); /*! @@ -91,9 +90,9 @@ std::vector getVarDim( int fid, const std::string& var ); * @details This function reads a variable with the given name from the file * @param fid Handle to the open file * @param var Variable to read -*/ + */ template -Array getVar( int fid, const std::string& var ); +Array getVar( int fid, const std::string &var ); /*! @@ -104,10 +103,10 @@ Array getVar( int fid, const std::string& var ); * @param start Starting corner for the read * @param count Number of elements to read * @param stride Stride size for the read -*/ + */ template -Array getVar( int fid, const std::string& var, const std::vector& start, - const std::vector& count, const std::vector& stride ); +Array getVar( int fid, const std::string &var, const std::vector &start, + const std::vector &count, const std::vector &stride ); /*! @@ -115,27 +114,29 @@ Array getVar( int fid, const std::string& var, const std::vector& sta * @details This function reads an attribute with the given name from the file * @param fid Handle to the open file * @param att Attribute to read -*/ + */ template -Array getAtt( int fid, const std::string& att ); +Array getAtt( int fid, const std::string &att ); /*! * @brief Write the dimensions - * @details This function writes the grid dimensions to netcdf. + * @details This function writes the grid dimensions to netcdf. * @param fid Handle to the open file -*/ -std::vector defDim( int fid, const std::vector& names, const std::vector& dims ); + */ +std::vector defDim( + int fid, const std::vector &names, const std::vector &dims ); /*! * @brief Write a variable - * @details This function writes a variable to netcdf. + * @details This function writes a variable to netcdf. * @param fid Handle to the open file -*/ + */ template -void write( int fid, const std::string& var, const std::vector& dimids, const Array& data, const RankInfoStruct& rank_info ); +void write( int fid, const std::string &var, const std::vector &dimids, + const Array &data, const RankInfoStruct &rank_info ); -}; // netcdf namespace +}; // namespace netcdf #endif diff --git a/IO/silo.cpp b/IO/silo.cpp index ddf3646a..6b6a2c39 100644 --- a/IO/silo.cpp +++ b/IO/silo.cpp @@ -1,6 +1,6 @@ #include "IO/silo.h" -#include "common/Utilities.h" #include "common/MPI.h" +#include "common/Utilities.h" #include "ProfilerApp.h" @@ -10,14 +10,13 @@ #include - namespace silo { /**************************************************** -* Open/close a file * -****************************************************/ -DBfile* open( const std::string& filename, FileMode mode ) + * Open/close a file * + ****************************************************/ +DBfile *open( const std::string &filename, FileMode mode ) { DBfile *fid = nullptr; if ( mode == CREATE ) { @@ -29,18 +28,15 @@ DBfile* open( const std::string& filename, FileMode mode ) } return fid; } -void close( DBfile* fid ) -{ - DBClose( fid ); -} +void close( DBfile *fid ) { DBClose( fid ); } /**************************************************** -* Helper functions * -****************************************************/ -VariableDataType varDataType( DBfile *fid, const std::string& name ) + * Helper functions * + ****************************************************/ +VariableDataType varDataType( DBfile *fid, const std::string &name ) { - auto type = DBGetVarType( fid, name.c_str() ); + auto type = DBGetVarType( fid, name.c_str() ); VariableDataType type2 = VariableDataType::UNKNOWN; if ( type == DB_DOUBLE ) type2 = VariableDataType::DOUBLE; @@ -53,58 +49,57 @@ VariableDataType varDataType( DBfile *fid, const std::string& name ) /**************************************************** -* Write/read a uniform mesh to silo * -****************************************************/ -void readUniformMesh( DBfile* fid, const std::string& meshname, - std::vector& range, std::vector& N ) + * Write/read a uniform mesh to silo * + ****************************************************/ +void readUniformMesh( + DBfile *fid, const std::string &meshname, std::vector &range, std::vector &N ) { - DBquadmesh* mesh = DBGetQuadmesh( fid, meshname.c_str() ); - int ndim = mesh->ndims; - range.resize(2*ndim); - N.resize(ndim); - for (int d=0; ddims[d]-1; - range[2*d+0] = mesh->min_extents[d]; - range[2*d+1] = mesh->max_extents[d]; + DBquadmesh *mesh = DBGetQuadmesh( fid, meshname.c_str() ); + int ndim = mesh->ndims; + range.resize( 2 * ndim ); + N.resize( ndim ); + for ( int d = 0; d < ndim; d++ ) { + N[d] = mesh->dims[d] - 1; + range[2 * d + 0] = mesh->min_extents[d]; + range[2 * d + 1] = mesh->max_extents[d]; } DBFreeQuadmesh( mesh ); } /**************************************************** -* Write a multimesh * -****************************************************/ -void writeMultiMesh( DBfile* fid, const std::string& meshname, - const std::vector& meshNames, - const std::vector& meshTypes ) + * Write a multimesh * + ****************************************************/ +void writeMultiMesh( DBfile *fid, const std::string &meshname, + const std::vector &meshNames, const std::vector &meshTypes ) { - std::vector meshnames(meshNames.size()); + std::vector meshnames( meshNames.size() ); for ( size_t i = 0; i < meshNames.size(); ++i ) meshnames[i] = (char *) meshNames[i].c_str(); std::string tree_name = meshname + "_tree"; DBoptlist *optList = DBMakeOptlist( 1 ); DBAddOption( optList, DBOPT_MRGTREE_NAME, (char *) tree_name.c_str() ); - DBPutMultimesh( fid, meshname.c_str(), meshNames.size(), meshnames.data(), (int*) meshTypes.data(), nullptr ); + DBPutMultimesh( fid, meshname.c_str(), meshNames.size(), meshnames.data(), + (int *) meshTypes.data(), nullptr ); DBFreeOptlist( optList ); } /**************************************************** -* Write a multivariable * -****************************************************/ -void writeMultiVar( DBfile* fid, const std::string& varname, - const std::vector& varNames, - const std::vector& varTypes ) + * Write a multivariable * + ****************************************************/ +void writeMultiVar( DBfile *fid, const std::string &varname, + const std::vector &varNames, const std::vector &varTypes ) { - std::vector varnames(varNames.size(),nullptr); - for (size_t j=0; j(varNames[j].c_str()); - DBPutMultivar( fid, varname.c_str(), varNames.size(), varnames.data(), (int*) varTypes.data(), nullptr ); + std::vector varnames( varNames.size(), nullptr ); + for ( size_t j = 0; j < varNames.size(); j++ ) + varnames[j] = const_cast( varNames[j].c_str() ); + DBPutMultivar( + fid, varname.c_str(), varNames.size(), varnames.data(), (int *) varTypes.data(), nullptr ); } - -}; // silo namespace +}; // namespace silo #else diff --git a/IO/silo.h b/IO/silo.h index 40a023d7..309746f3 100644 --- a/IO/silo.h +++ b/IO/silo.h @@ -1,29 +1,34 @@ #ifndef SILO_INTERFACE #define SILO_INTERFACE +#include #include #include -#include #include "common/Array.h" -#include "common/MPI.h" #include "common/Communication.h" +#include "common/MPI.h" #ifdef USE_SILO - #include +#include #else - typedef int DBfile; +typedef int DBfile; #endif - namespace silo { enum FileMode { READ, WRITE, CREATE }; -enum class VariableType : int { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 }; +enum class VariableType : int { + NodeVariable = 1, + EdgeVariable = 2, + SurfaceVariable = 2, + VolumeVariable = 3, + NullVariable = 0 +}; enum class VariableDataType { DOUBLE, FLOAT, INT, UNKNOWN }; @@ -34,16 +39,16 @@ enum class VariableDataType { DOUBLE, FLOAT, INT, UNKNOWN }; * @param[in] filename File to open * @param[in] mode Open the file for reading or writing * @return This function returns a handle to the file -*/ -DBfile* open( const std::string& filename, FileMode mode ); + */ +DBfile *open( const std::string &filename, FileMode mode ); /*! * @brief Close silo file * @details This function closes a silo file * @param[in] fid Handle to the open file -*/ -void close( DBfile* fid ); + */ +void close( DBfile *fid ); /*! @@ -51,8 +56,8 @@ void close( DBfile* fid ); * @details This function returns the type of variable data * @param[in] fid Handle to the open file * @param[in] name Name of variable -*/ -VariableDataType varDataType( DBfile *dbfile, const std::string& name ); + */ +VariableDataType varDataType( DBfile *dbfile, const std::string &name ); /*! @@ -61,9 +66,9 @@ VariableDataType varDataType( DBfile *dbfile, const std::string& name ); * @param[in] fid Handle to the open file * @param[in] varname Variable name * @param[in] data Data to write -*/ + */ template -void write( DBfile* fid, const std::string& varname, const std::vector& data ); +void write( DBfile *fid, const std::string &varname, const std::vector &data ); /*! @@ -72,9 +77,9 @@ void write( DBfile* fid, const std::string& varname, const std::vector& da * @param[in] fid Handle to the open file * @param[in] varname Variable name * @return Data read -*/ + */ template -std::vector read( DBfile* fid, const std::string& varname ); +std::vector read( DBfile *fid, const std::string &varname ); /*! @@ -84,10 +89,10 @@ std::vector read( DBfile* fid, const std::string& varname ); * @param[in] meshname Mesh name * @param[in] range Range of mesh { xmin, xmax, ymin, ymax, zmin, zmax } * @param[in] N Number of cells in each direction -*/ + */ template -void writeUniformMesh( DBfile* fid, const std::string& meshname, - const std::array& range, const std::array& N ); +void writeUniformMesh( DBfile *fid, const std::string &meshname, + const std::array &range, const std::array &N ); /*! @@ -97,9 +102,9 @@ void writeUniformMesh( DBfile* fid, const std::string& meshname, * @param[in] meshname Mesh name * @param[out] range Range of mesh { xmin, xmax, ymin, ymax, zmin, zmax } * @param[out] N Number of cells in each direction -*/ -void readUniformMesh( DBfile* fid, const std::string& meshname, - std::vector& range, std::vector& N ); + */ +void readUniformMesh( + DBfile *fid, const std::string &meshname, std::vector &range, std::vector &N ); /*! @@ -111,10 +116,11 @@ void readUniformMesh( DBfile* fid, const std::string& meshname, * @param[in] varname Variable name * @param[in] data Variable data * @param[in] type Variable type -*/ -template< int NDIM, class TYPE > -void writeUniformMeshVariable( DBfile* fid, const std::string& meshname, const std::array& N, - const std::string& varname, const Array& data, VariableType type ); + */ +template +void writeUniformMeshVariable( DBfile *fid, const std::string &meshname, + const std::array &N, const std::string &varname, const Array &data, + VariableType type ); /*! @@ -123,9 +129,9 @@ void writeUniformMeshVariable( DBfile* fid, const std::string& meshname, const s * @param[in] fid Handle to the open file * @param[in] varname Variable name * @return Variable data -*/ + */ template -Array readUniformMeshVariable( DBfile* fid, const std::string& varname ); +Array readUniformMeshVariable( DBfile *fid, const std::string &varname ); /*! @@ -136,10 +142,10 @@ Array readUniformMeshVariable( DBfile* fid, const std::string& varname ); * @param[in] ndim Number of dimensions * @param[in] N Number of points * @param[in] coords Coordinates of the points -*/ + */ template -void writePointMesh( DBfile* fid, const std::string& meshname, - int ndim, int N, const TYPE *coords[] ); +void writePointMesh( + DBfile *fid, const std::string &meshname, int ndim, int N, const TYPE *coords[] ); /*! @@ -147,10 +153,10 @@ void writePointMesh( DBfile* fid, const std::string& meshname, * @details This function reads a pointmesh from silo * @param[in] fid Handle to the open file * @param[in] meshname Mesh name - * @return Returns the coordinates as a N x ndim array -*/ + * @return Returns the coordinates as a N x ndim array + */ template -Array readPointMesh( DBfile* fid, const std::string& meshname ); +Array readPointMesh( DBfile *fid, const std::string &meshname ); /*! @@ -160,10 +166,10 @@ Array readPointMesh( DBfile* fid, const std::string& meshname ); * @param[in] meshname Mesh name * @param[in] varname Variable name * @param[in] data Variable data -*/ + */ template -void writePointMeshVariable( DBfile* fid, const std::string& meshname, - const std::string& varname, const Array& data ); +void writePointMeshVariable( + DBfile *fid, const std::string &meshname, const std::string &varname, const Array &data ); /*! @@ -172,9 +178,9 @@ void writePointMeshVariable( DBfile* fid, const std::string& meshname, * @param[in] fid Handle to the open file * @param[in] varname Variable name * @return Variable data -*/ + */ template -Array readPointMeshVariable( DBfile* fid, const std::string& varname ); +Array readPointMeshVariable( DBfile *fid, const std::string &varname ); /*! @@ -188,10 +194,10 @@ Array readPointMeshVariable( DBfile* fid, const std::string& varname ); * @param[in] coords Coordinates of the points * @param[in] N_tri Number of triangles * @param[in] tri Coordinates of the points -*/ + */ template -void writeTriMesh( DBfile* fid, const std::string& meshname, - int ndim, int ndim_tri, int N, const TYPE *coords[], int N_tri, const int *tri[] ); +void writeTriMesh( DBfile *fid, const std::string &meshname, int ndim, int ndim_tri, int N, + const TYPE *coords[], int N_tri, const int *tri[] ); /*! @@ -201,9 +207,9 @@ void writeTriMesh( DBfile* fid, const std::string& meshname, * @param[in] meshname Mesh name * @param[in] coords Coordinates of the points * @param[in] tri Coordinates of the points -*/ + */ template -void readTriMesh( DBfile* fid, const std::string& meshname, Array& coords, Array& tri ); +void readTriMesh( DBfile *fid, const std::string &meshname, Array &coords, Array &tri ); /*! @@ -215,10 +221,10 @@ void readTriMesh( DBfile* fid, const std::string& meshname, Array& coords, * @param[in] varname Variable name * @param[in] data Variable data * @param[in] type Variable type -*/ + */ template -void writeTriMeshVariable( DBfile* fid, int ndim, const std::string& meshname, - const std::string& varname, const Array& data, VariableType type ); +void writeTriMeshVariable( DBfile *fid, int ndim, const std::string &meshname, + const std::string &varname, const Array &data, VariableType type ); /*! @@ -227,9 +233,9 @@ void writeTriMeshVariable( DBfile* fid, int ndim, const std::string& meshname, * @param[in] fid Handle to the open file * @param[in] varname Variable name * @return Variable data -*/ + */ template -Array readTriMeshVariable( DBfile* fid, const std::string& varname ); +Array readTriMeshVariable( DBfile *fid, const std::string &varname ); /*! @@ -239,10 +245,9 @@ Array readTriMeshVariable( DBfile* fid, const std::string& varname ); * @param[in] meshname Mesh name * @param[in] subMeshNames Names of the sub meshes in the form "filename:meshname" * @param[in] subMeshTypes Type of each submesh -*/ -void writeMultiMesh( DBfile* fid, const std::string& meshname, - const std::vector& subMeshNames, - const std::vector& subMeshTypes ); + */ +void writeMultiMesh( DBfile *fid, const std::string &meshname, + const std::vector &subMeshNames, const std::vector &subMeshTypes ); /*! @@ -255,14 +260,12 @@ void writeMultiMesh( DBfile* fid, const std::string& meshname, * @param[in] subVarTypes Type of each submesh * @param[in] ndim Dimension of variable (used to determine suffix) * @param[in] nvar Number of subvariables (used to determine suffix) -*/ -void writeMultiVar( DBfile* fid, const std::string& varname, - const std::vector& subVarNames, - const std::vector& subVarTypes ); + */ +void writeMultiVar( DBfile *fid, const std::string &varname, + const std::vector &subVarNames, const std::vector &subVarTypes ); -}; // silo namespace +}; // namespace silo #endif #include "IO/silo.hpp" - diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8df4e6bd..2405b463 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -35,7 +35,7 @@ ADD_LBPM_EXECUTABLE( GenerateSphereTest ) #ADD_LBPM_EXECUTABLE( BlobAnalysis ) #ADD_LBPM_EXECUTABLE( BlobIdentify ) #ADD_LBPM_EXECUTABLE( BlobIdentifyParallel ) -#ADD_LBPM_EXECUTABLE( convertIO ) +ADD_LBPM_EXECUTABLE( convertIO ) #ADD_LBPM_EXECUTABLE( DataAggregator ) #ADD_LBPM_EXECUTABLE( BlobAnalyzeParallel )( ADD_LBPM_EXECUTABLE( lbpm_minkowski_scalar ) diff --git a/tests/TestWriter.cpp b/tests/TestWriter.cpp index 4030930c..9b7e381c 100644 --- a/tests/TestWriter.cpp +++ b/tests/TestWriter.cpp @@ -159,7 +159,7 @@ void testWriter( const std::string& format, std::vector& mes // Test the simple read interface bool pass = true; for ( const auto& timestep : timesteps ) { - auto data = IO::readData( path, timestep ); + auto data = IO::readData( path, timestep, comm.getRank() ); pass = pass && data.size() == meshData.size(); for ( size_t i=0; i #include -#include "common/MPI_Helpers.h" -#include "common/Communication.h" +#include "common/MPI.h" #include "common/Utilities.h" #include "IO/Mesh.h" #include "IO/Reader.h" @@ -16,74 +15,56 @@ int main(int argc, char **argv) { - // Initialize MPI - Utilities::startup( argc, argv ); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); - Utilities::setErrorHandlers(); - PROFILE_ENABLE(2); - PROFILE_ENABLE_TRACE(); - PROFILE_START("Main"); - { // Limit scope + // Initialize MPI + Utilities::startup( argc, argv ); + Utilities::setErrorHandlers(); + PROFILE_ENABLE(2); + PROFILE_ENABLE_TRACE(); + PROFILE_START("Main"); - // Get inputs - if ( argc != 3 ) { - std::cerr << "Error calling convertIO:\n"; - std::cerr << " convertIO input_file format\n"; - return -1; - } - std::string filename = argv[1]; - std::string format = argv[2]; - std::string path = IO::getPath( filename ); + { // Limit scope - // Read the timesteps - auto timesteps = IO::readTimesteps( filename, "old" ); - - // Loop through the timesteps, reading/writing the data - IO::initialize( "", format, false ); - for ( auto timestep : timesteps ) { - - // Read the list of MeshDatabase - auto databases = IO::getMeshList( path, timestep ); - - // Build the MeshDataStruct - std::vector meshData(databases.size()); - - // Loop through the database - int i = 0; - PROFILE_START("Read"); - for ( const auto& database : databases ) { - - // Read the appropriate mesh domain - ASSERT( (int) database.domains.size() == nprocs ); - meshData[i].meshName = database.name; - meshData[i].mesh = IO::getMesh( path, timestep, database, rank ); - - // Read the variables - for ( auto var : database.variables ) { - auto varData = IO::getVariable( path, timestep, database, rank, var.name ); - IO::reformatVariable( *meshData[i].mesh, *varData ); - meshData[i].vars.push_back( varData ); - } - - i++; + Utilities::MPI comm( MPI_COMM_WORLD ); + // Get inputs + if ( argc != 5 ) { + std::cerr << "Error calling convertIO:\n"; + std::cerr << " convertIO \n"; + return -1; } - MPI_Barrier(comm); - PROFILE_STOP("Read"); + std::string path_in = argv[1]; + std::string format_in = argv[2]; + std::string path_out = argv[3]; + std::string format_out = argv[4]; - // Save the mesh data to a new file - PROFILE_START("Write"); - IO::writeData( timestep, meshData, MPI_COMM_WORLD ); - MPI_Barrier(comm); - PROFILE_STOP("Write"); - } + // Check that we have enough ranks to load and write the data + // This is really only a bottleneck for the writer + int N_domains = IO::maxDomains( path_in, format_in, comm ); + ASSERT( comm.getSize() == N_domains ); - } // Limit scope - PROFILE_STOP("Main"); - PROFILE_SAVE("convertData",true); - comm.barrier(); - Utilities::shutdown(); - return 0; + // Read the timesteps + auto timesteps = IO::readTimesteps( path_in, format_in ); + + // Loop through the timesteps, reading/writing the data + IO::initialize( path_out, format_out, false ); + for ( auto timestep : timesteps ) { + + // Set the domain to read (needs to be the current rank for the writer to be valid) + int domain = comm.getRank(); + + // Get the maximum number of domains for the + auto data = IO::readData( path_in, timestep, domain ); + + // Save the mesh data to a new file + IO::writeData( timestep, data, comm ); + + } + + } // Limit scope + + // shutdown + PROFILE_STOP("Main"); + PROFILE_SAVE("convertData",true); + Utilities::shutdown(); + return 0; } From dfa97a013251e78728ead35f8e18820a91b2b6b9 Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Wed, 17 Mar 2021 13:34:38 -0400 Subject: [PATCH 18/32] Fixing minor build error without timer --- CMakeLists.txt | 2 +- cmake/Find_TIMER.cmake | 41 ++++++++++++++++++++++++++++++----------- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d479391..8f500927 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -150,7 +150,7 @@ IF ( NOT ONLY_BUILD_DOCS ) CONFIGURE_NETCDF() CONFIGURE_SILO() CONFIGURE_LBPM() - CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" ) + CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" FALSE ) CONFIGURE_LINE_COVERAGE() # Set the external library link list SET( EXTERNAL_LIBS ${EXTERNAL_LIBS} ${TIMER_LIBS} ) diff --git a/cmake/Find_TIMER.cmake b/cmake/Find_TIMER.cmake index 7ebc7aea..a4d7bdb2 100644 --- a/cmake/Find_TIMER.cmake +++ b/cmake/Find_TIMER.cmake @@ -4,7 +4,7 @@ # CONFIGURE_TIMER( DEFAULT_USE_TIMER NULL_TIMER_DIR ) # This function assumes that USE_TIMER is set to indicate if the timer should be used # If USE_TIMER is set, TIMER_DIRECTORY specifies the install path for the timer -# If USE_TIMER is not set we will create a summy timer that does nothing. +# If USE_TIMER is not set we will create a dummy timer that does nothing. # The input argument DEFAULT_USE_TIMER specifies if the timer library is included by default. # The input argument NULL_TIMER_DIR specifies the location to install the dummy timer. # If it is an empty string, the default install path "${CMAKE_CURRENT_BINARY_DIR}/null_timer" is used. @@ -13,7 +13,7 @@ # TIMER_CXXFLAGS - C++ flags for the timer library # TIMER_LDFLAGS - Linker flags to link the timer library # TIMER_LDLIBS - Linker libraries to link the timer library -FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR ) +FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR QUIET ) # Determine if we want to use the timer utility CHECK_ENABLE_FLAG( USE_TIMER ${DEFAULT_USE_TIMER} ) SET( TIMER_INCLUDE ) @@ -33,20 +33,23 @@ FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR ) FIND_LIBRARY( TIMER_LIBS NAMES timerutility PATHS ${TIMER_DIRECTORY}/lib NO_DEFAULT_PATH ) SET( TIMER_INCLUDE ${TIMER_DIRECTORY}/include ) SET( TIMER_CXXFLAGS "-DUSE_TIMER -I${TIMER_DIRECTORY}/include" ) - SET( TIMER_LDFLAGS -L${TIMER_DIRECTORY}/lib ) - SET( TIMER_LDLIBS -ltimerutility ) + SET( TIMER_LDFLAGS ) + SET( TIMER_LDLIBS "${TIMER_LIBS}" ) ELSE() MESSAGE( FATAL_ERROR "Default search for TIMER is not yet supported. Use -D TIMER_DIRECTORY=" ) ENDIF() - SET(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} "${TIMER_DIRECTORY}/lib" PARENT_SCOPE ) + SET( CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} "${TIMER_DIRECTORY}/lib" PARENT_SCOPE ) INCLUDE_DIRECTORIES( "${TIMER_INCLUDE}" ) ADD_DEFINITIONS( -DUSE_TIMER ) - MESSAGE( "Using timer utility" ) - MESSAGE( " TIMER_LIBRARIES = ${TIMER_LIBS}" ) + IF ( NOT QUIET ) + MESSAGE( STATUS "Using timer utility" ) + MESSAGE( STATUS " TIMER_LIBRARIES = ${TIMER_LIBS}" ) + ENDIF() ELSE() IF ( "${NULL_TIMER_DIR}" STREQUAL "" ) SET( NULL_TIMER_DIR "${CMAKE_CURRENT_BINARY_DIR}/null_timer" ) ENDIF() + # Write ProfilerApp.h FILE(WRITE "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_START(...) do {} while(0)\n" ) FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_STOP(...) do {} while(0)\n" ) FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_START2(...) do {} while(0)\n" ) @@ -61,9 +64,25 @@ FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR ) FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_DISABLE_TRACE() do {} while(0)\n" ) FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_ENABLE_MEMORY() do {} while(0)\n" ) FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_DISABLE_MEMORY() do {} while(0)\n" ) + # Write MemoryApp.h + FILE(WRITE "${NULL_TIMER_DIR}/MemoryApp.h" "#include \n" ) + FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "class MemoryApp final {\n" ) + FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "public:\n" ) + FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " struct MemoryStats {\n" ) + FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " size_t bytes_new, bytes_delete, N_new, N_delete, tot_bytes_used, system_memory, stack_used, stack_size;\n" ) + FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " MemoryStats() { memset(this,0,sizeof(MemoryStats)); }\n" ) + FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " };\n" ) + FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline void print( std::ostream& ) {}\n" ) + FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline size_t getMemoryUsage() { return 0; }\n" ) + FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline size_t getTotalMemoryUsage() { return 0; }\n" ) + FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline size_t getSystemMemory() { return 0; }\n" ) + FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline MemoryStats getMemoryStats() { return MemoryStats(); }\n" ) + FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "};\n" ) SET( TIMER_INCLUDE "${NULL_TIMER_DIR}" ) INCLUDE_DIRECTORIES( "${TIMER_INCLUDE}" ) - MESSAGE( "Disabling timer utility" ) + IF ( NOT QUIET ) + MESSAGE( STATUS "Disabling timer utility" ) + ENDIF() ENDIF() SET( TIMER_INCLUDE "${TIMER_INCLUDE}" PARENT_SCOPE ) SET( TIMER_CXXFLAGS "${TIMER_CXXFLAGS}" PARENT_SCOPE ) @@ -88,12 +107,12 @@ MACRO( CHECK_ENABLE_FLAG FLAG DEFAULT ) SET( ${FLAG} ${DEFAULT} ) ELSEIF( ${FLAG} STREQUAL "" ) SET( ${FLAG} ${DEFAULT} ) - ELSEIF( ( ${${FLAG}} STREQUAL "false" ) OR ( ${${FLAG}} STREQUAL "0" ) OR ( ${${FLAG}} STREQUAL "OFF" ) ) + ELSEIF( ( ${${FLAG}} STREQUAL "FALSE" ) OR ( ${${FLAG}} STREQUAL "false" ) OR ( ${${FLAG}} STREQUAL "0" ) OR ( ${${FLAG}} STREQUAL "OFF" ) ) SET( ${FLAG} 0 ) - ELSEIF( ( ${${FLAG}} STREQUAL "true" ) OR ( ${${FLAG}} STREQUAL "1" ) OR ( ${${FLAG}} STREQUAL "ON" ) ) + ELSEIF( ( ${${FLAG}} STREQUAL "TRUE" ) OR ( ${${FLAG}} STREQUAL "true" ) OR ( ${${FLAG}} STREQUAL "1" ) OR ( ${${FLAG}} STREQUAL "ON" ) ) SET( ${FLAG} 1 ) ELSE() - MESSAGE( "Bad value for ${FLAG} (${${FLAG}}); use true or false" ) + MESSAGE( FATAL_ERROR "Bad value for ${FLAG} (${${FLAG}}); use true or false" ) ENDIF () ENDMACRO() From 59b7b9a0fee0b5fe498b6c56776d37c0c82ecc45 Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Thu, 18 Mar 2021 09:41:39 -0400 Subject: [PATCH 19/32] Fixing issues with enums and the summary files --- IO/Mesh.cpp | 170 ++++++++++++++++-- IO/Mesh.h | 31 +++- IO/MeshDatabase.cpp | 76 ++++---- IO/MeshDatabase.h | 9 +- IO/Reader.cpp | 18 +- IO/Writer.cpp | 90 +++++----- IO/silo.cpp | 16 +- IO/silo.h | 17 +- IO/silo.hpp | 4 +- tests/TestWriter.cpp | 400 ++++++++++++++++++++++--------------------- 10 files changed, 480 insertions(+), 351 deletions(-) diff --git a/IO/Mesh.cpp b/IO/Mesh.cpp index 91c78e03..9966bf52 100644 --- a/IO/Mesh.cpp +++ b/IO/Mesh.cpp @@ -1,4 +1,5 @@ #include "Mesh.h" +#include "IO/IOHelpers.h" #include "common/Utilities.h" #include @@ -28,16 +29,23 @@ Mesh::~Mesh() {} /**************************************************** * MeshDataStruct * ****************************************************/ -bool MeshDataStruct::check() const +#define checkResult( pass, msg ) \ + do { \ + if ( !( pass ) ) { \ + if ( abort ) \ + ERROR( msg ); \ + return false; \ + } \ + } while ( 0 ) +bool MeshDataStruct::check( bool abort ) const { - bool pass = mesh != nullptr; for ( const auto &var : vars ) { - pass = pass && static_cast( var->type ) >= 1 && static_cast( var->type ) <= 3; - pass = pass && !var->data.empty(); - } - if ( !pass ) { - std::cerr << "Invalid variable detected\n"; - return false; + checkResult( var->type == VariableType::NodeVariable || + var->type == VariableType::EdgeVariable || + var->type == VariableType::SurfaceVariable || + var->type == VariableType::VolumeVariable, + "Invalid data type" ); + checkResult( !var->data.empty(), "Variable data is empty" ); } const std::string &meshClass = mesh->className(); if ( meshClass == "PointList" ) { @@ -45,7 +53,9 @@ bool MeshDataStruct::check() const ASSERT( mesh2 ); for ( const auto &var : vars ) { if ( var->type == IO::VariableType::NodeVariable ) { - pass = pass && var->data.size() == ArraySize( mesh2->points.size(), var->dim ); + size_t N_points = mesh2->points.size(); + checkResult( var->data.size( 0 ) == N_points, "sizeof NodeVariable" ); + checkResult( var->data.size( 1 ) == var->dim, "sizeof NodeVariable" ); } else if ( var->type == IO::VariableType::EdgeVariable ) { ERROR( "Invalid type for PointList" ); } else if ( var->type == IO::VariableType::SurfaceVariable ) { @@ -61,15 +71,16 @@ bool MeshDataStruct::check() const ASSERT( mesh2 ); for ( const auto &var : vars ) { if ( var->type == IO::VariableType::NodeVariable ) { - pass = pass && - var->data.size() == ArraySize( mesh2->vertices->points.size(), var->dim ); + size_t N_points = mesh2->vertices->points.size(); + checkResult( var->data.size( 0 ) == N_points, "sizeof NodeVariable" ); + checkResult( var->data.size( 1 ) == var->dim, "sizeof NodeVariable" ); } else if ( var->type == IO::VariableType::EdgeVariable ) { ERROR( "Not finished" ); } else if ( var->type == IO::VariableType::SurfaceVariable ) { ERROR( "Not finished" ); } else if ( var->type == IO::VariableType::VolumeVariable ) { - pass = pass && var->data.size( 0 ) == mesh2->A.size() && - var->data.size( 1 ) == var->dim; + checkResult( var->data.size( 0 ) == mesh2->A.size(), "sizeof VolumeVariable" ); + checkResult( var->data.size( 1 ) == var->dim, "sizeof VolumeVariable" ); } else { ERROR( "Invalid variable type" ); } @@ -90,14 +101,16 @@ bool MeshDataStruct::check() const } else { ERROR( "Invalid variable type" ); } - if ( var->data.size() == ArraySize( varSize[0] * varSize[1] * varSize[2], varSize[3] ) ) + if ( var->data.size( 0 ) == varSize[0] * varSize[1] * varSize[2] && + var->data.size( 1 ) == varSize[3] ) var->data.resize( varSize ); - pass = pass && var->data.size() == varSize; + for ( int d = 0; d < 4; d++ ) + checkResult( var->data.size( d ) == varSize[d], "DomainMesh Variable" ); } } else { ERROR( "Unknown mesh class: " + mesh->className() ); } - return pass; + return true; } @@ -479,4 +492,129 @@ std::shared_ptr getTriList( std::shared_ptr mesh ) } +/**************************************************** + * Convert enum values * + ****************************************************/ +std::string getString( VariableType type ) +{ + if ( type == VariableType::NodeVariable ) + return "node"; + else if ( type == VariableType::EdgeVariable ) + return "edge"; + else if ( type == VariableType::SurfaceVariable ) + return "face"; + else if ( type == VariableType::VolumeVariable ) + return "cell"; + else if ( type == VariableType::NullVariable ) + return "null"; + else + ERROR( "Invalid type" ); + return ""; +} +VariableType getVariableType( const std::string &type_in ) +{ + auto type = deblank( type_in ); + if ( type == "node" ) + return VariableType::NodeVariable; + else if ( type == "edge" || type == "1" ) + return VariableType::EdgeVariable; + else if ( type == "face" ) + return VariableType::SurfaceVariable; + else if ( type == "cell" || type == "3" ) + return VariableType::VolumeVariable; + else if ( type == "null" ) + return VariableType::NullVariable; + else + ERROR( "Invalid type: " + type ); + return VariableType::NullVariable; +} +std::string getString( DataType type ) +{ + if ( type == DataType::Double ) + return "double"; + else if ( type == DataType::Float ) + return "float"; + else if ( type == DataType::Int ) + return "int"; + else if ( type == DataType::Null ) + return "null"; + else + ERROR( "Invalid type" ); + return ""; +} +DataType getDataType( const std::string &type_in ) +{ + auto type = deblank( type_in ); + if ( type == "double" ) + return DataType::Double; + else if ( type == "float" ) + return DataType::Float; + else if ( type == "int" ) + return DataType::Int; + else if ( type == "null" ) + return DataType::Null; + else + ERROR( "Invalid type: " + type ); + return DataType::Null; +} +std::string getString( MeshType type ) +{ + if ( type == MeshType::PointMesh ) + return "PointMesh"; + else if ( type == MeshType::SurfaceMesh ) + return "SurfaceMesh"; + else if ( type == MeshType::VolumeMesh ) + return "VolumeMesh"; + else if ( type == MeshType::Unknown ) + return "unknown"; + else + ERROR( "Invalid type" ); + return ""; +} +MeshType getMeshType( const std::string &type_in ) +{ + auto type = deblank( type_in ); + if ( type == "PointMesh" || type == "1" ) + return MeshType::PointMesh; + else if ( type == "SurfaceMesh" || type == "2" ) + return MeshType::SurfaceMesh; + else if ( type == "VolumeMesh" || type == "3" ) + return MeshType::VolumeMesh; + else if ( type == "unknown" || type == "-1" ) + return MeshType::Unknown; + else + ERROR( "Invalid type: " + type ); + return MeshType::Unknown; +} +std::string getString( FileFormat type ) +{ + if ( type == FileFormat::OLD ) + return "old"; + else if ( type == FileFormat::NEW ) + return "new"; + else if ( type == FileFormat::NEW_SINGLE ) + return "new(single)"; + else if ( type == FileFormat::SILO ) + return "silo"; + else + ERROR( "Invalid type" ); + return ""; +} +FileFormat getFileFormat( const std::string &type_in ) +{ + auto type = deblank( type_in ); + if ( type == "old" || type == "1" ) + return FileFormat::OLD; + else if ( type == "new" || type == "2" ) + return FileFormat::NEW; + else if ( type == "new(single)" || type == "3" ) + return FileFormat::NEW_SINGLE; + else if ( type == "silo" || type == "4" ) + return FileFormat::SILO; + else + ERROR( "Invalid type: " + type ); + return FileFormat::SILO; +} + + } // namespace IO diff --git a/IO/Mesh.h b/IO/Mesh.h index a60e14c9..a420f95d 100644 --- a/IO/Mesh.h +++ b/IO/Mesh.h @@ -14,15 +14,28 @@ namespace IO { -//! Possible variable types -enum class VariableType : unsigned char { - NodeVariable = 1, - EdgeVariable = 2, - SurfaceVariable = 3, - VolumeVariable = 4, - NullVariable = 0 +//! Enums to define types +enum class VariableType { + NodeVariable, + EdgeVariable, + SurfaceVariable, + VolumeVariable, + NullVariable }; -enum class DataType : unsigned char { Double = 1, Float = 2, Int = 2, Null = 0 }; +enum class DataType { Double, Float, Int, Null }; +enum class MeshType { PointMesh, SurfaceMesh, VolumeMesh, Unknown }; +enum class FileFormat { OLD, NEW, NEW_SINGLE, SILO }; + + +//! Convert enums to/from strings (more future-proof than static_cast) +std::string getString( VariableType ); +std::string getString( DataType ); +std::string getString( MeshType ); +std::string getString( FileFormat ); +VariableType getVariableType( const std::string & ); +DataType getDataType( const std::string & ); +MeshType getMeshType( const std::string & ); +FileFormat getFileFormat( const std::string & ); /*! \class Mesh @@ -216,7 +229,7 @@ struct MeshDataStruct { //! Empty constructor MeshDataStruct() : precision( DataType::Double ) {} //! Check the data - bool check() const; + bool check( bool abort = true ) const; }; diff --git a/IO/MeshDatabase.cpp b/IO/MeshDatabase.cpp index 70b9acc3..63702c7b 100644 --- a/IO/MeshDatabase.cpp +++ b/IO/MeshDatabase.cpp @@ -13,38 +13,31 @@ #include -// MeshType -template<> -size_t packsize( const IO::MeshType &rhs ) -{ - return sizeof( IO::MeshType ); -} -template<> -void pack( const IO::MeshType &rhs, char *buffer ) -{ - memcpy( buffer, &rhs, sizeof( IO::MeshType ) ); -} -template<> -void unpack( IO::MeshType &data, const char *buffer ) -{ - memcpy( &data, buffer, sizeof( IO::MeshType ) ); -} -// Variable::VariableType -template<> -size_t packsize( const IO::VariableType &rhs ) -{ - return sizeof( IO::VariableType ); -} -template<> -void pack( const IO::VariableType &rhs, char *buffer ) -{ - memcpy( buffer, &rhs, sizeof( IO::VariableType ) ); -} -template<> -void unpack( IO::VariableType &data, const char *buffer ) -{ - memcpy( &data, buffer, sizeof( IO::VariableType ) ); -} +// Default pack/unpack +// clang-format off +#define INSTANTIATE_PACK( TYPE ) \ + template<> \ + size_t packsize( const TYPE &rhs ) \ + { \ + return sizeof( TYPE ); \ + } \ + template<> \ + void pack( const TYPE &rhs, char *buffer ) \ + { \ + memcpy( buffer, &rhs, sizeof( IO::MeshType ) ); \ + } \ + template<> \ + void unpack( TYPE &data, const char *buffer ) \ + { \ + memcpy( &data, buffer, sizeof( IO::MeshType ) ); \ + } +INSTANTIATE_PACK( IO::VariableType ) +INSTANTIATE_PACK( IO::DataType ) +INSTANTIATE_PACK( IO::MeshType ) +INSTANTIATE_PACK( IO::FileFormat ) +// clang-format on + + // DatabaseEntry template<> size_t packsize( const IO::DatabaseEntry &rhs ) @@ -327,8 +320,7 @@ std::vector gatherAll( // Return the results std::vector data2( data.size() ); size_t i = 0; - for ( std::map::iterator it = data.begin(); it != data.end(); - ++it, ++i ) + for ( auto it = data.begin(); it != data.end(); ++it, ++i ) data2[i] = it->second; PROFILE_STOP( "gatherAll-unpack", 2 ); PROFILE_STOP( "gatherAll" ); @@ -343,19 +335,19 @@ void write( const std::vector &meshes, const std::string &filename FILE *fid = fopen( filename.c_str(), "wb" ); for ( size_t i = 0; i < meshes.size(); i++ ) { fprintf( fid, "%s\n", meshes[i].name.c_str() ); - fprintf( fid, " type: %i\n", static_cast( meshes[i].type ) ); + fprintf( fid, " type: %s\n", getString( meshes[i].type ).data() ); fprintf( fid, " meshClass: %s\n", meshes[i].meshClass.c_str() ); - fprintf( fid, " format: %i\n", static_cast( meshes[i].format ) ); + fprintf( fid, " format: %s\n", getString( meshes[i].format ).data() ); for ( size_t j = 0; j < meshes[i].domains.size(); j++ ) fprintf( fid, " domain: %s\n", meshes[i].domains[j].write().c_str() ); fprintf( fid, " variables: " ); for ( size_t j = 0; j < meshes[i].variables.size(); j++ ) { const VariableDatabase &var = meshes[i].variables[j]; - fprintf( fid, "%s|%i|%i; ", var.name.c_str(), static_cast( var.type ), var.dim ); + fprintf( fid, "%s|%s|%i; ", var.name.data(), getString( var.type ).data(), var.dim ); } fprintf( fid, "\n" ); - std::map, DatabaseEntry>::const_iterator it; - for ( it = meshes[i].variable_data.begin(); it != meshes[i].variable_data.end(); ++it ) { + for ( auto it = meshes[i].variable_data.begin(); it != meshes[i].variable_data.end(); + ++it ) { const char *domain = it->first.first.c_str(); const char *variable = it->first.second.c_str(); fprintf( @@ -386,9 +378,9 @@ std::vector read( const std::string &filename ) name.resize( name.size() - 1 ); meshes.back().name = name; } else if ( strncmp( line, " format:", 10 ) == 0 ) { - meshes.back().format = static_cast( atoi( &line[10] ) ); + meshes.back().format = getFileFormat( &line[10] ); } else if ( strncmp( line, " type:", 8 ) == 0 ) { - meshes.back().type = static_cast( atoi( &line[8] ) ); + meshes.back().type = getMeshType( &line[8] ); } else if ( strncmp( line, " meshClass:", 13 ) == 0 ) { meshes.back().meshClass = deblank( std::string( &line[13] ) ); } else if ( strncmp( line, " domain:", 10 ) == 0 ) { @@ -402,7 +394,7 @@ std::vector read( const std::string &filename ) std::vector tmp = splitList( variables[i].c_str(), '|' ); ASSERT( tmp.size() == 3 ); mesh.variables[i].name = tmp[0]; - mesh.variables[i].type = static_cast( atoi( tmp[1].c_str() ) ); + mesh.variables[i].type = getVariableType( tmp[1] ); mesh.variables[i].dim = atoi( tmp[2].c_str() ); } } else if ( strncmp( line, " variable(", 12 ) == 0 ) { diff --git a/IO/MeshDatabase.h b/IO/MeshDatabase.h index 0dfd968c..508f85d8 100644 --- a/IO/MeshDatabase.h +++ b/IO/MeshDatabase.h @@ -13,13 +13,6 @@ namespace IO { -class Mesh; - - -//! Enum to identify mesh type -// enum class MeshType : char { PointMesh=1, SurfaceMesh=2, VolumeMesh=3, Unknown=-1 }; -enum class MeshType { PointMesh = 1, SurfaceMesh = 2, VolumeMesh = 3, Unknown = -1 }; - //! Helper struct for containing offsets for the mesh info struct DatabaseEntry { @@ -56,7 +49,7 @@ struct MeshDatabase { std::string name; //!< Name of the mesh MeshType type; //!< Mesh type std::string meshClass; //!< Mesh class - unsigned char format; //!< Data format (1: old, 2: new, 3: new (single), 4: silo) + FileFormat format; //!< Data format (1: old, 2: new, 3: new (single), 4: silo) std::vector domains; //!< List of the domains std::vector variables; //!< List of the variables std::map variable_data; //!< Data for the variables diff --git a/IO/Reader.cpp b/IO/Reader.cpp index e5fae5bc..e63b8dd3 100644 --- a/IO/Reader.cpp +++ b/IO/Reader.cpp @@ -155,7 +155,7 @@ std::shared_ptr IO::getMesh( const std::string &path, const std::strin { PROFILE_START( "getMesh" ); std::shared_ptr mesh; - if ( meshDatabase.format == 1 ) { + if ( meshDatabase.format == FileFormat::OLD ) { // Old format (binary doubles) std::string filename = path + "/" + timestep + "/" + meshDatabase.domains[domain].file; FILE *fid = fopen( filename.c_str(), "rb" ); @@ -206,7 +206,8 @@ std::shared_ptr IO::getMesh( const std::string &path, const std::strin ERROR( "Unknown mesh type" ); } delete[] data; - } else if ( meshDatabase.format == 2 ) { + } else if ( meshDatabase.format == FileFormat::NEW || + meshDatabase.format == FileFormat::NEW_SINGLE ) { const DatabaseEntry &database = meshDatabase.domains[domain]; std::string filename = path + "/" + timestep + "/" + database.file; FILE *fid = fopen( filename.c_str(), "rb" ); @@ -233,7 +234,7 @@ std::shared_ptr IO::getMesh( const std::string &path, const std::strin } mesh->unpack( std::pair( bytes, data ) ); delete[] data; - } else if ( meshDatabase.format == 4 ) { + } else if ( meshDatabase.format == FileFormat::SILO ) { // Reading a silo file #ifdef USE_SILO const DatabaseEntry &database = meshDatabase.domains[domain]; @@ -301,12 +302,11 @@ std::shared_ptr IO::getVariable( const std::string &path, const st const MeshDatabase &meshDatabase, int domain, const std::string &variable ) { std::pair key( meshDatabase.domains[domain].name, variable ); - std::map, DatabaseEntry>::const_iterator it; - it = meshDatabase.variable_data.find( key ); + auto it = meshDatabase.variable_data.find( key ); if ( it == meshDatabase.variable_data.end() ) return std::shared_ptr(); std::shared_ptr var; - if ( meshDatabase.format == 2 ) { + if ( meshDatabase.format == FileFormat::NEW || meshDatabase.format == FileFormat::NEW_SINGLE ) { const DatabaseEntry &database = it->second; std::string filename = path + "/" + timestep + "/" + database.file; FILE *fid = fopen( filename.c_str(), "rb" ); @@ -318,13 +318,13 @@ std::shared_ptr IO::getVariable( const std::string &path, const st std::vector values = splitList( &line[i2 + 1], ',' ); ASSERT( values.size() == 5 ); int dim = atoi( values[0].c_str() ); - int type = atoi( values[1].c_str() ); + auto type = values[1]; size_t N = atol( values[2].c_str() ); size_t bytes = atol( values[3].c_str() ); std::string precision = values[4]; var = std::shared_ptr( new IO::Variable() ); var->dim = dim; - var->type = static_cast( type ); + var->type = getVariableType( type ); var->name = variable; var->data.resize( N, dim ); if ( precision == "double" ) { @@ -334,7 +334,7 @@ std::shared_ptr IO::getVariable( const std::string &path, const st ERROR( "Format not implimented" ); } fclose( fid ); - } else if ( meshDatabase.format == 4 ) { + } else if ( meshDatabase.format == FileFormat::SILO ) { // Reading a silo file #ifdef USE_SILO const auto &database = meshDatabase.domains[domain]; diff --git a/IO/Writer.cpp b/IO/Writer.cpp index d3f9d991..051db47d 100644 --- a/IO/Writer.cpp +++ b/IO/Writer.cpp @@ -110,7 +110,7 @@ static std::vector writeMeshesOrigFormat( mesh_entry.name = meshData[i].meshName; mesh_entry.type = meshType( *mesh ); mesh_entry.meshClass = meshData[i].mesh->className(); - mesh_entry.format = 1; + mesh_entry.format = IO::FileFormat::OLD; IO::DatabaseEntry domain; domain.name = domainname; domain.file = filename; @@ -171,7 +171,7 @@ static std::vector writeMeshesOrigFormat( // Create the database entry for the mesh data static IO::MeshDatabase getDatabase( - const std::string &filename, const IO::MeshDataStruct &mesh, int format, int rank ) + const std::string &filename, const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank ) { char domainname[100]; sprintf( domainname, "%s_%05i", mesh.meshName.c_str(), rank ); @@ -209,8 +209,8 @@ static IO::MeshDatabase getDatabase( // Write a mesh (and variables) to a file -static IO::MeshDatabase write_domain( - FILE *fid, const std::string &filename, const IO::MeshDataStruct &mesh, int format, int rank ) +static IO::MeshDatabase write_domain( FILE *fid, const std::string &filename, + const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank ) { const int level = 0; // Create the MeshDatabase @@ -225,19 +225,17 @@ static IO::MeshDatabase write_domain( delete[]( char * ) data.second; // Write the variables for ( size_t i = 0; i < mesh.vars.size(); i++ ) { + ASSERT( mesh.vars[i]->type != IO::VariableType::NullVariable ); std::pair key( domain.name, mesh.vars[i]->name ); - IO::DatabaseEntry &variable = database.variable_data[key]; - variable.offset = ftell( fid ); - int dim = mesh.vars[i]->dim; - int type = static_cast( mesh.vars[i]->type ); - size_t N = mesh.vars[i]->data.length(); - if ( type == static_cast( IO::VariableType::NullVariable ) ) { - ERROR( "Variable type not set" ); - } - size_t N_mesh = mesh.mesh->numberPointsVar( mesh.vars[i]->type ); + auto &variable = database.variable_data[key]; + variable.offset = ftell( fid ); + int dim = mesh.vars[i]->dim; + auto type = getString( mesh.vars[i]->type ); + size_t N = mesh.vars[i]->data.length(); + size_t N_mesh = mesh.mesh->numberPointsVar( mesh.vars[i]->type ); ASSERT( N == dim * N_mesh ); - fprintf( fid, "Var: %s-%05i-%s: %i, %i, %lu, %lu, double\n", database.name.c_str(), rank, - variable.name.c_str(), dim, type, N_mesh, N * sizeof( double ) ); + fprintf( fid, "Var: %s-%05i-%s: %i, %s, %lu, %lu, double\n", database.name.c_str(), rank, + variable.name.c_str(), dim, type.data(), N_mesh, N * sizeof( double ) ); fwrite( mesh.vars[i]->data.data(), sizeof( double ), N, fid ); fprintf( fid, "\n" ); } @@ -259,7 +257,7 @@ static void writeSiloPointMesh( z[i] = points[i].z; } const TYPE *coords[] = { x.data(), y.data(), z.data() }; - silo::writePointMesh( fid, meshname, 3, points.size(), coords ); + IO::silo::writePointMesh( fid, meshname, 3, points.size(), coords ); } static void writeSiloPointList( DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database ) @@ -281,19 +279,19 @@ static void writeSiloPointList( z[i] = points[i].z; } const double *coords[] = { x.data(), y.data(), z.data() }; - silo::writePointMesh( fid, meshname, 3, points.size(), coords ); + IO::silo::writePointMesh( fid, meshname, 3, points.size(), coords ); for ( size_t i = 0; i < meshData.vars.size(); i++ ) { const IO::Variable &var = *meshData.vars[i]; if ( var.precision == IO::DataType::Double ) { - silo::writePointMeshVariable( fid, meshname, var.name, var.data ); + IO::silo::writePointMeshVariable( fid, meshname, var.name, var.data ); } else if ( var.precision == IO::DataType::Float ) { Array data2( var.data.size() ); data2.copy( var.data ); - silo::writePointMeshVariable( fid, meshname, var.name, data2 ); + IO::silo::writePointMeshVariable( fid, meshname, var.name, data2 ); } else if ( var.precision == IO::DataType::Int ) { Array data2( var.data.size() ); data2.copy( var.data ); - silo::writePointMeshVariable( fid, meshname, var.name, data2 ); + IO::silo::writePointMeshVariable( fid, meshname, var.name, data2 ); } else { ERROR( "Unsupported format" ); } @@ -312,7 +310,7 @@ static void writeSiloTriMesh( DBfile *fid, const IO::TriMesh &mesh, const std::s } const TYPE *coords[] = { x.data(), y.data(), z.data() }; const int *tri[] = { mesh.A.data(), mesh.B.data(), mesh.C.data() }; - silo::writeTriMesh( fid, meshname, 3, 2, points.size(), coords, mesh.A.size(), tri ); + IO::silo::writeTriMesh( fid, meshname, 3, 2, points.size(), coords, mesh.A.size(), tri ); } static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct &meshData, const IO::TriMesh &mesh, IO::MeshDatabase database ) @@ -327,17 +325,16 @@ static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct &meshData, } for ( size_t i = 0; i < meshData.vars.size(); i++ ) { const IO::Variable &var = *meshData.vars[i]; - auto type = static_cast( var.type ); if ( var.precision == IO::DataType::Double ) { - silo::writeTriMeshVariable( fid, 3, meshname, var.name, var.data, type ); + IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, var.data, var.type ); } else if ( var.precision == IO::DataType::Float ) { Array data2( var.data.size() ); data2.copy( var.data ); - silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, type ); + IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, var.type ); } else if ( var.precision == IO::DataType::Int ) { Array data2( var.data.size() ); data2.copy( var.data ); - silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, type ); + IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, var.type ); } else { ERROR( "Unsupported format" ); } @@ -367,30 +364,29 @@ static void writeSiloDomainMesh( ( info.kz + 1 ) * mesh.Lz / info.nz }; std::array N = { mesh.nx, mesh.ny, mesh.nz }; auto meshname = database.domains[0].name; - silo::writeUniformMesh<3>( fid, meshname, range, N ); - silo::write( + IO::silo::writeUniformMesh<3>( fid, meshname, range, N ); + IO::silo::write( fid, meshname + "_rankinfo", { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz } ); for ( size_t i = 0; i < meshData.vars.size(); i++ ) { const auto &var = *meshData.vars[i]; - auto type = static_cast( var.type ); if ( var.precision == IO::DataType::Double ) { - silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, var.data, type ); + IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, var.data, var.type ); } else if ( var.precision == IO::DataType::Float ) { Array data2( var.data.size() ); data2.copy( var.data ); - silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, type ); + IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, var.type ); } else if ( var.precision == IO::DataType::Int ) { Array data2( var.data.size() ); data2.copy( var.data ); - silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, type ); + IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, var.type ); } else { ERROR( "Unsupported format" ); } } } // Write a mesh (and variables) to a file -static IO::MeshDatabase write_domain_silo( - DBfile *fid, const std::string &filename, const IO::MeshDataStruct &mesh, int format, int rank ) +static IO::MeshDatabase write_domain_silo( DBfile *fid, const std::string &filename, + const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank ) { // Create the MeshDatabase auto database = getDatabase( filename, mesh, format, rank ); @@ -432,7 +428,7 @@ std::pair getSiloMeshType( const std::string &meshClass ) void writeSiloSummary( const std::vector &meshes_written, const std::string &filename ) { - auto fid = silo::open( filename, silo::CREATE ); + auto fid = IO::silo::open( filename, IO::silo::CREATE ); for ( const auto &data : meshes_written ) { auto type = getSiloMeshType( data.meshClass ); std::vector meshTypes( data.domains.size(), type.first ); @@ -440,22 +436,23 @@ void writeSiloSummary( std::vector meshNames; for ( const auto &tmp : data.domains ) meshNames.push_back( tmp.file + ":" + tmp.name ); - silo::writeMultiMesh( fid, data.name, meshNames, meshTypes ); + IO::silo::writeMultiMesh( fid, data.name, meshNames, meshTypes ); for ( const auto &variable : data.variables ) { std::vector varnames; for ( const auto &tmp : data.domains ) varnames.push_back( tmp.file + ":" + variable.name ); - silo::writeMultiVar( fid, variable.name, varnames, varTypes ); + IO::silo::writeMultiVar( fid, variable.name, varnames, varTypes ); } } - silo::close( fid ); + IO::silo::close( fid ); } #endif // Write the mesh data in the new format static std::vector writeMeshesNewFormat( - const std::vector &meshData, const std::string &path, int format, int rank ) + const std::vector &meshData, const std::string &path, IO::FileFormat format, + int rank ) { std::vector meshes_written; char filename[100], fullpath[200]; @@ -473,19 +470,20 @@ static std::vector writeMeshesNewFormat( // Write the mesh data to silo static std::vector writeMeshesSilo( - const std::vector &meshData, const std::string &path, int format, int rank ) + const std::vector &meshData, const std::string &path, IO::FileFormat format, + int rank ) { #ifdef USE_SILO std::vector meshes_written; char filename[100], fullpath[200]; sprintf( filename, "%05i.silo", rank ); sprintf( fullpath, "%s/%s", path.c_str(), filename ); - auto fid = silo::open( fullpath, silo::CREATE ); + auto fid = IO::silo::open( fullpath, IO::silo::CREATE ); for ( size_t i = 0; i < meshData.size(); i++ ) { auto mesh = meshData[i].mesh; meshes_written.push_back( write_domain_silo( fid, filename, meshData[i], format, rank ) ); } - silo::close( fid ); + IO::silo::close( fid ); return meshes_written; #else NULL_USE( meshData ); @@ -509,10 +507,8 @@ void IO::writeData( const std::string &subdir, const std::vector -namespace silo { +namespace IO::silo { /**************************************************** @@ -34,16 +34,16 @@ void close( DBfile *fid ) { DBClose( fid ); } /**************************************************** * Helper functions * ****************************************************/ -VariableDataType varDataType( DBfile *fid, const std::string &name ) +DataType varDataType( DBfile *fid, const std::string &name ) { - auto type = DBGetVarType( fid, name.c_str() ); - VariableDataType type2 = VariableDataType::UNKNOWN; + auto type = DBGetVarType( fid, name.c_str() ); + DataType type2 = DataType::Null; if ( type == DB_DOUBLE ) - type2 = VariableDataType::DOUBLE; + type2 = DataType::Double; else if ( type == DB_FLOAT ) - type2 = VariableDataType::FLOAT; + type2 = DataType::Float; else if ( type == DB_INT ) - type2 = VariableDataType::INT; + type2 = DataType::Int; return type2; } @@ -99,7 +99,7 @@ void writeMultiVar( DBfile *fid, const std::string &varname, } -}; // namespace silo +}; // namespace IO::silo #else diff --git a/IO/silo.h b/IO/silo.h index 309746f3..5e1068fe 100644 --- a/IO/silo.h +++ b/IO/silo.h @@ -5,6 +5,7 @@ #include #include +#include "IO/Mesh.h" #include "common/Array.h" #include "common/Communication.h" #include "common/MPI.h" @@ -17,21 +18,11 @@ typedef int DBfile; #endif -namespace silo { +namespace IO::silo { enum FileMode { READ, WRITE, CREATE }; -enum class VariableType : int { - NodeVariable = 1, - EdgeVariable = 2, - SurfaceVariable = 2, - VolumeVariable = 3, - NullVariable = 0 -}; - -enum class VariableDataType { DOUBLE, FLOAT, INT, UNKNOWN }; - /*! * @brief Open silo file @@ -57,7 +48,7 @@ void close( DBfile *fid ); * @param[in] fid Handle to the open file * @param[in] name Name of variable */ -VariableDataType varDataType( DBfile *dbfile, const std::string &name ); +DataType varDataType( DBfile *dbfile, const std::string &name ); /*! @@ -265,7 +256,7 @@ void writeMultiVar( DBfile *fid, const std::string &varname, const std::vector &subVarNames, const std::vector &subVarTypes ); -}; // namespace silo +}; // namespace IO::silo #endif #include "IO/silo.hpp" diff --git a/IO/silo.hpp b/IO/silo.hpp index 1e17aa5c..b76ebd28 100644 --- a/IO/silo.hpp +++ b/IO/silo.hpp @@ -13,7 +13,7 @@ #include -namespace silo { +namespace IO::silo { /**************************************************** @@ -413,7 +413,7 @@ Array readTriMeshVariable( DBfile *fid, const std::string &varname ) } -}; // namespace silo +}; // namespace IO::silo #endif diff --git a/tests/TestWriter.cpp b/tests/TestWriter.cpp index 9b7e381c..652c3d4c 100644 --- a/tests/TestWriter.cpp +++ b/tests/TestWriter.cpp @@ -1,115 +1,117 @@ +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include "common/UnitTest.h" -#include "common/Utilities.h" -#include "common/MPI.h" #include "IO/MeshDatabase.h" #include "IO/Reader.h" #include "IO/Writer.h" #include "ProfilerApp.h" +#include "common/MPI.h" +#include "common/UnitTest.h" +#include "common/Utilities.h" -inline bool approx_equal( const Point& A, const Point& B ) +inline bool approx_equal( const Point &A, const Point &B ) { - double tol = 1e-7*sqrt(A.x*A.x+A.y*A.y+A.z*A.z); - return fabs(A.x-B.x)<=tol && fabs(A.y-B.y)<=tol && fabs(A.z-B.z)<=tol; + double tol = 1e-7 * sqrt( A.x * A.x + A.y * A.y + A.z * A.z ); + return fabs( A.x - B.x ) <= tol && fabs( A.y - B.y ) <= tol && fabs( A.z - B.z ) <= tol; } -inline bool approx_equal( const double& A, const double& B ) +inline bool approx_equal( const double &A, const double &B ) { - return fabs(A-B) <= std::max(1e-7*fabs(A+B),1e-20); + return fabs( A - B ) <= std::max( 1e-7 * fabs( A + B ), 1e-20 ); } -inline double distance( const Point& p ) -{ - return sqrt(p.x*p.x+p.y*p.y+p.z*p.z); -} +inline double distance( const Point &p ) { return sqrt( p.x * p.x + p.y * p.y + p.z * p.z ); } -bool checkMesh( const std::vector& meshData, const std::string& format, std::shared_ptr mesh ) +bool checkMesh( const std::vector &meshData, const std::string &format, + std::shared_ptr mesh ) { // Get direct access to the meshes used to test the reader - const auto pointmesh = dynamic_cast( meshData[0].mesh.get() ); - const auto trimesh = dynamic_cast( meshData[1].mesh.get() ); - const auto trilist = dynamic_cast( meshData[2].mesh.get() ); - const auto domain = dynamic_cast( meshData[3].mesh.get() ); - const size_t N_tri = trimesh->A.size(); + const auto pointmesh = dynamic_cast( meshData[0].mesh.get() ); + const auto trimesh = dynamic_cast( meshData[1].mesh.get() ); + const auto trilist = dynamic_cast( meshData[2].mesh.get() ); + const auto domain = dynamic_cast( meshData[3].mesh.get() ); + const size_t N_tri = trimesh->A.size(); if ( mesh->className() == "pointmesh" ) { // Check the pointmesh - auto pmesh = IO::getPointList(mesh); - if ( pmesh.get()==NULL ) + auto pmesh = IO::getPointList( mesh ); + if ( pmesh.get() == NULL ) return false; if ( pmesh->points.size() != pointmesh->points.size() ) return false; } if ( mesh->className() == "trimesh" || mesh->className() == "trilist" ) { // Check the trimesh/trilist - auto mesh1 = IO::getTriMesh(mesh); - auto mesh2 = IO::getTriList(mesh); - if ( mesh1.get()==NULL || mesh2.get()==NULL ) - return false; - if ( mesh1->A.size()!=N_tri || mesh1->B.size()!=N_tri || mesh1->C.size()!=N_tri || - mesh2->A.size()!=N_tri || mesh2->B.size()!=N_tri || mesh2->C.size()!=N_tri ) - return false; - const std::vector& P1 = mesh1->vertices->points; - const std::vector& A1 = mesh1->A; - const std::vector& B1 = mesh1->B; - const std::vector& C1 = mesh1->C; - const std::vector& A2 = mesh2->A; - const std::vector& B2 = mesh2->B; - const std::vector& C2 = mesh2->C; - const std::vector& A = trilist->A; - const std::vector& B = trilist->B; - const std::vector& C = trilist->C; - for (size_t i=0; iA.size() != N_tri || mesh1->B.size() != N_tri || mesh1->C.size() != N_tri || + mesh2->A.size() != N_tri || mesh2->B.size() != N_tri || mesh2->C.size() != N_tri ) + return false; + const std::vector &P1 = mesh1->vertices->points; + const std::vector &A1 = mesh1->A; + const std::vector &B1 = mesh1->B; + const std::vector &C1 = mesh1->C; + const std::vector &A2 = mesh2->A; + const std::vector &B2 = mesh2->B; + const std::vector &C2 = mesh2->C; + const std::vector &A = trilist->A; + const std::vector &B = trilist->B; + const std::vector &C = trilist->C; + for ( size_t i = 0; i < N_tri; i++ ) { + if ( !approx_equal( P1[A1[i]], A[i] ) || !approx_equal( P1[B1[i]], B[i] ) || + !approx_equal( P1[C1[i]], C[i] ) ) return false; - if ( !approx_equal(A2[i],A[i]) || !approx_equal(B2[i],B[i]) || !approx_equal(C2[i],C[i]) ) + if ( !approx_equal( A2[i], A[i] ) || !approx_equal( B2[i], B[i] ) || + !approx_equal( C2[i], C[i] ) ) return false; } } - if ( mesh->className() == "domain" && format!="old" ) { + if ( mesh->className() == "domain" && format != "old" ) { // Check the domain mesh - const IO::DomainMesh& mesh1 = *std::dynamic_pointer_cast(mesh); - if ( mesh1.nprocx!=domain->nprocx || mesh1.nprocy!=domain->nprocy || mesh1.nprocz!=domain->nprocz ) + const IO::DomainMesh &mesh1 = *std::dynamic_pointer_cast( mesh ); + if ( mesh1.nprocx != domain->nprocx || mesh1.nprocy != domain->nprocy || + mesh1.nprocz != domain->nprocz ) return false; - if ( mesh1.nx!=domain->nx || mesh1.ny!=domain->ny || mesh1.nz!=domain->nz ) + if ( mesh1.nx != domain->nx || mesh1.ny != domain->ny || mesh1.nz != domain->nz ) return false; - if ( mesh1.Lx!=domain->Lx || mesh1.Ly!=domain->Ly || mesh1.Lz!=domain->Lz ) + if ( mesh1.Lx != domain->Lx || mesh1.Ly != domain->Ly || mesh1.Lz != domain->Lz ) return false; } return true; } -bool checkVar( const std::string& format, std::shared_ptr mesh, +bool checkVar( const std::string &format, std::shared_ptr mesh, std::shared_ptr variable1, std::shared_ptr variable2 ) { - if ( format=="new" ) + if ( format == "new" ) IO::reformatVariable( *mesh, *variable2 ); - bool pass = true; - const IO::Variable& var1 = *variable1; - const IO::Variable& var2 = *variable2; - pass = var1.name == var2.name; - pass = pass && var1.dim == var2.dim; - pass = pass && var1.type == var2.type; - pass = pass && var1.data.length() == var2.data.length(); + bool pass = true; + const IO::Variable &var1 = *variable1; + const IO::Variable &var2 = *variable2; + pass = var1.name == var2.name; + pass = pass && var1.dim == var2.dim; + pass = pass && var1.type == var2.type; + pass = pass && var1.data.length() == var2.data.length(); if ( pass ) { - for (size_t m=0; m& meshData, UnitTest& ut ) +void testWriter( + const std::string &format, std::vector &meshData, UnitTest &ut ) { PROFILE_SCOPED( path, 0, timer ); @@ -124,20 +126,20 @@ void testWriter( const std::string& format, std::vector& mes // Get the format std::string format2 = format; - auto precision = IO::DataType::Double; + auto precision = IO::DataType::Double; if ( format == "silo-double" ) { - format2 = "silo"; + format2 = "silo"; precision = IO::DataType::Double; } else if ( format == "silo-float" ) { - format2 = "silo"; + format2 = "silo"; precision = IO::DataType::Float; } // Set the precision for the variables - for ( auto& data : meshData ) { + for ( auto &data : meshData ) { data.precision = precision; - for ( auto& var : data.vars ) + for ( auto &var : data.vars ) var->precision = precision; } @@ -150,18 +152,18 @@ void testWriter( const std::string& format, std::vector& mes // Get a list of the timesteps auto timesteps = IO::readTimesteps( path, format2 ); - if ( timesteps.size()==2 ) - ut.passes(format+": Corrent number of timesteps"); + if ( timesteps.size() == 2 ) + ut.passes( format + ": Corrent number of timesteps" ); else - ut.failure(format+": Incorrent number of timesteps"); + ut.failure( format + ": Incorrent number of timesteps" ); // Test the simple read interface bool pass = true; - for ( const auto& timestep : timesteps ) { + for ( const auto ×tep : timesteps ) { auto data = IO::readData( path, timestep, comm.getRank() ); - pass = pass && data.size() == meshData.size(); - for ( size_t i=0; i& mes // Test reading each mesh domain - for ( const auto& timestep : timesteps ) { + for ( const auto ×tep : timesteps ) { // Load the list of meshes and check its size - auto databaseList = IO::getMeshList(path,timestep); - if ( databaseList.size()==meshData.size() ) - ut.passes(format+": Corrent number of meshes found"); + auto databaseList = IO::getMeshList( path, timestep ); + if ( databaseList.size() == meshData.size() ) + ut.passes( format + ": Corrent number of meshes found" ); else - ut.failure(format+": Incorrent number of meshes found"); + ut.failure( format + ": Incorrent number of meshes found" ); // Check the number of domains for each mesh - for ( const auto& database : databaseList ) { + for ( const auto &database : databaseList ) { int N_domains = database.domains.size(); if ( N_domains != nprocs ) { ut.failure( format + ": Incorrent number of domains for mesh" ); @@ -188,8 +190,8 @@ void testWriter( const std::string& format, std::vector& mes } // For each domain, load the mesh and check its data bool pass = true; - for (int k=0; k& mes } } if ( pass ) { - ut.passes(format+": Mesh \"" + database.name + "\" loaded correctly"); + ut.passes( format + ": Mesh \"" + database.name + "\" loaded correctly" ); } else { - ut.failure(format+": Mesh \"" + database.name + "\" did not load correctly"); + ut.failure( format + ": Mesh \"" + database.name + "\" did not load correctly" ); continue; } // Load the variables and check their data - if ( format=="old" ) - continue; // Old format does not support variables - const IO::MeshDataStruct* mesh0 = nullptr; - for (size_t k=0; kvars.size(); v++) { - PROFILE_START(format+"-read-getVariable"); - auto variable = IO::getVariable(path,timestep,database,k,mesh0->vars[v]->name); + for ( int k = 0; k < N_domains; k++ ) { + auto mesh = IO::getMesh( path, timestep, database, k ); + for ( size_t v = 0; v < mesh0->vars.size(); v++ ) { + PROFILE_START( format + "-read-getVariable" ); + auto variable = + IO::getVariable( path, timestep, database, k, mesh0->vars[v]->name ); pass = checkVar( format, mesh, mesh0->vars[v], variable ); if ( pass ) { - ut.passes(format+": Variable \"" + variable->name + "\" matched"); + ut.passes( format + ": Variable \"" + variable->name + "\" matched" ); } else { - ut.failure(format+": Variable \"" + variable->name + "\" did not match"); + ut.failure( + format + ": Variable \"" + variable->name + "\" did not match" ); break; } } @@ -233,157 +237,161 @@ void testWriter( const std::string& format, std::vector& mes // Main -int main(int argc, char **argv) +int main( int argc, char **argv ) { Utilities::startup( argc, argv ); Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); + int rank = comm.getRank(); int nprocs = comm.getSize(); - Utilities::setAbortBehavior(true,2); + Utilities::setAbortBehavior( true, 2 ); Utilities::setErrorHandlers(); UnitTest ut; // Create some points const int N_points = 8; - const int N_tri = 12; - double x[8] = { 0, 1, 0, 1, 0, 1, 0, 1 }; - double y[8] = { 0, 0, 1, 1, 0, 0, 1, 1 }; - double z[8] = { 0, 0, 0, 0, 1, 1, 1, 1 }; - int tri[N_tri][3] = { - {0,1,3}, {0,3,2}, {4,5,7}, {4,7,6}, // z faces - {0,1,4}, {1,4,5}, {2,3,6}, {3,6,7}, // y faces - {0,2,4}, {2,4,6}, {1,3,5}, {3,5,7} // x faces + const int N_tri = 12; + double x[8] = { 0, 1, 0, 1, 0, 1, 0, 1 }; + double y[8] = { 0, 0, 1, 1, 0, 0, 1, 1 }; + double z[8] = { 0, 0, 0, 0, 1, 1, 1, 1 }; + int tri[N_tri][3] = { + { 0, 1, 3 }, { 0, 3, 2 }, { 4, 5, 7 }, { 4, 7, 6 }, // z faces + { 0, 1, 4 }, { 1, 4, 5 }, { 2, 3, 6 }, { 3, 6, 7 }, // y faces + { 0, 2, 4 }, { 2, 4, 6 }, { 1, 3, 5 }, { 3, 5, 7 } // x faces }; // Create the meshes - auto set1 = std::make_shared(N_points); - for (int i=0; i( N_points ); + for ( int i = 0; i < N_points; i++ ) { set1->points[i].x = x[i]; set1->points[i].y = y[i]; set1->points[i].z = z[i]; } - auto trimesh = std::make_shared(N_tri,set1); - for (int i=0; i( N_tri, set1 ); + for ( int i = 0; i < N_tri; i++ ) { trimesh->A[i] = tri[i][0]; trimesh->B[i] = tri[i][1]; trimesh->C[i] = tri[i][2]; } - auto trilist = std::make_shared(*trimesh); - for (int i=0; iA[i],A) || !approx_equal(trilist->B[i],B) || !approx_equal(trilist->C[i],C) ) - { - printf("Failed to create trilist\n"); + auto trilist = std::make_shared( *trimesh ); + for ( int i = 0; i < N_tri; i++ ) { + Point A( x[tri[i][0]], y[tri[i][0]], z[tri[i][0]] ); + Point B( x[tri[i][1]], y[tri[i][1]], z[tri[i][1]] ); + Point C( x[tri[i][2]], y[tri[i][2]], z[tri[i][2]] ); + if ( !approx_equal( trilist->A[i], A ) || !approx_equal( trilist->B[i], B ) || + !approx_equal( trilist->C[i], C ) ) { + printf( "Failed to create trilist\n" ); return -1; } } RankInfoStruct rank_data( rank, nprocs, 1, 1 ); - auto domain = std::make_shared(rank_data,6,7,8,1.0,1.0,1.0); + auto domain = std::make_shared( rank_data, 6, 7, 8, 1.0, 1.0, 1.0 ); // Create the variables const auto NodeVar = IO::VariableType::NodeVariable; const auto VolVar = IO::VariableType::VolumeVariable; - auto set_node_mag = std::make_shared(1,NodeVar,"Node_set_mag"); - auto set_node_vec = std::make_shared(3,NodeVar,"Node_set_vec"); - auto list_node_mag = std::make_shared(1,NodeVar,"Node_list_mag"); - auto list_node_vec = std::make_shared(3,NodeVar,"Node_list_vec"); - auto point_node_mag = std::make_shared(1,NodeVar,"Node_point_mag"); - auto point_node_vec = std::make_shared(3,NodeVar,"Node_point_vec"); - auto domain_node_mag = std::make_shared(1,NodeVar,"Node_domain_mag"); - auto domain_node_vec = std::make_shared(3,NodeVar,"Node_domain_vec"); - auto set_cell_mag = std::make_shared(1,VolVar,"Cell_set_mag"); - auto set_cell_vec = std::make_shared(3,VolVar,"Cell_set_vec"); - auto list_cell_mag = std::make_shared(1,VolVar,"Cell_list_mag"); - auto list_cell_vec = std::make_shared(3,VolVar,"Cell_list_vec"); - auto domain_cell_mag = std::make_shared(1,VolVar,"Cell_domain_mag"); - auto domain_cell_vec = std::make_shared(3,VolVar,"Cell_domain_vec"); + auto set_node_mag = std::make_shared( 1, NodeVar, "Node_set_mag" ); + auto set_node_vec = std::make_shared( 3, NodeVar, "Node_set_vec" ); + auto list_node_mag = std::make_shared( 1, NodeVar, "Node_list_mag" ); + auto list_node_vec = std::make_shared( 3, NodeVar, "Node_list_vec" ); + auto point_node_mag = std::make_shared( 1, NodeVar, "Node_point_mag" ); + auto point_node_vec = std::make_shared( 3, NodeVar, "Node_point_vec" ); + auto domain_node_mag = std::make_shared( 1, NodeVar, "Node_domain_mag" ); + auto domain_node_vec = std::make_shared( 3, NodeVar, "Node_domain_vec" ); + auto set_cell_mag = std::make_shared( 1, VolVar, "Cell_set_mag" ); + auto set_cell_vec = std::make_shared( 3, VolVar, "Cell_set_vec" ); + auto list_cell_mag = std::make_shared( 1, VolVar, "Cell_list_mag" ); + auto list_cell_vec = std::make_shared( 3, VolVar, "Cell_list_vec" ); + auto domain_cell_mag = std::make_shared( 1, VolVar, "Cell_domain_mag" ); + auto domain_cell_vec = std::make_shared( 3, VolVar, "Cell_domain_vec" ); point_node_mag->data.resize( N_points ); point_node_vec->data.resize( N_points, 3 ); - for (int i=0; idata(i) = distance(set1->points[i]); - point_node_vec->data(i,0) = set1->points[i].x; - point_node_vec->data(i,1) = set1->points[i].y; - point_node_vec->data(i,2) = set1->points[i].z; + for ( int i = 0; i < N_points; i++ ) { + point_node_mag->data( i ) = distance( set1->points[i] ); + point_node_vec->data( i, 0 ) = set1->points[i].x; + point_node_vec->data( i, 1 ) = set1->points[i].y; + point_node_vec->data( i, 2 ) = set1->points[i].z; } set_node_mag->data = point_node_mag->data; set_node_vec->data = point_node_vec->data; - list_node_mag->data.resize( 3*N_tri ); - list_node_vec->data.resize( 3*N_tri, 3 ); - for (int i=0; idata(3*i+0) = distance(trilist->A[i]); - list_node_mag->data(3*i+1) = distance(trilist->B[i]); - list_node_mag->data(3*i+2) = distance(trilist->C[i]); - list_node_vec->data(3*i+0,0) = trilist->A[i].x; - list_node_vec->data(3*i+0,1) = trilist->A[i].y; - list_node_vec->data(3*i+0,2) = trilist->A[i].z; - list_node_vec->data(3*i+1,0) = trilist->B[i].x; - list_node_vec->data(3*i+1,1) = trilist->B[i].y; - list_node_vec->data(3*i+1,2) = trilist->B[i].z; - list_node_vec->data(3*i+2,0) = trilist->C[i].x; - list_node_vec->data(3*i+2,1) = trilist->C[i].y; - list_node_vec->data(3*i+2,2) = trilist->C[i].z; + list_node_mag->data.resize( 3 * N_tri ); + list_node_vec->data.resize( 3 * N_tri, 3 ); + for ( int i = 0; i < N_points; i++ ) { + list_node_mag->data( 3 * i + 0 ) = distance( trilist->A[i] ); + list_node_mag->data( 3 * i + 1 ) = distance( trilist->B[i] ); + list_node_mag->data( 3 * i + 2 ) = distance( trilist->C[i] ); + list_node_vec->data( 3 * i + 0, 0 ) = trilist->A[i].x; + list_node_vec->data( 3 * i + 0, 1 ) = trilist->A[i].y; + list_node_vec->data( 3 * i + 0, 2 ) = trilist->A[i].z; + list_node_vec->data( 3 * i + 1, 0 ) = trilist->B[i].x; + list_node_vec->data( 3 * i + 1, 1 ) = trilist->B[i].y; + list_node_vec->data( 3 * i + 1, 2 ) = trilist->B[i].z; + list_node_vec->data( 3 * i + 2, 0 ) = trilist->C[i].x; + list_node_vec->data( 3 * i + 2, 1 ) = trilist->C[i].y; + list_node_vec->data( 3 * i + 2, 2 ) = trilist->C[i].z; } - domain_node_mag->data.resize(domain->nx+1,domain->ny+1,domain->nz+1); - domain_node_vec->data.resize({(size_t)domain->nx+1,(size_t)domain->ny+1,(size_t)domain->nz+1,3}); - for (int i=0; inx+1; i++) { - for (int j=0; jny+1; j++) { - for (int k=0; knz+1; k++) { - domain_node_mag->data(i,j,k) = distance(Point(i,j,k)); - domain_node_vec->data(i,j,k,0) = Point(i,j,k).x; - domain_node_vec->data(i,j,k,1) = Point(i,j,k).y; - domain_node_vec->data(i,j,k,2) = Point(i,j,k).z; + domain_node_mag->data.resize( domain->nx + 1, domain->ny + 1, domain->nz + 1 ); + domain_node_vec->data.resize( + { (size_t) domain->nx + 1, (size_t) domain->ny + 1, (size_t) domain->nz + 1, 3 } ); + for ( int i = 0; i < domain->nx + 1; i++ ) { + for ( int j = 0; j < domain->ny + 1; j++ ) { + for ( int k = 0; k < domain->nz + 1; k++ ) { + domain_node_mag->data( i, j, k ) = distance( Point( i, j, k ) ); + domain_node_vec->data( i, j, k, 0 ) = Point( i, j, k ).x; + domain_node_vec->data( i, j, k, 1 ) = Point( i, j, k ).y; + domain_node_vec->data( i, j, k, 2 ) = Point( i, j, k ).z; } } } set_cell_mag->data.resize( N_tri ); set_cell_vec->data.resize( N_tri, 3 ); - for (int i=0; idata(i) = i; - set_cell_vec->data(i,0) = 3*i+0; - set_cell_vec->data(i,1) = 3*i+1; - set_cell_vec->data(i,2) = 3*i+2; + for ( int i = 0; i < N_tri; i++ ) { + set_cell_mag->data( i ) = i; + set_cell_vec->data( i, 0 ) = 3 * i + 0; + set_cell_vec->data( i, 1 ) = 3 * i + 1; + set_cell_vec->data( i, 2 ) = 3 * i + 2; } list_cell_mag->data = set_cell_mag->data; list_cell_vec->data = set_cell_vec->data; - domain_cell_mag->data.resize(domain->nx,domain->ny,domain->nz); - domain_cell_vec->data.resize({(size_t)domain->nx,(size_t)domain->ny,(size_t)domain->nz,3}); - for (int i=0; inx; i++) { - for (int j=0; jny; j++) { - for (int k=0; knz; k++) { - domain_cell_mag->data(i,j,k) = distance(Point(i,j,k)); - domain_cell_vec->data(i,j,k,0) = Point(i,j,k).x; - domain_cell_vec->data(i,j,k,1) = Point(i,j,k).y; - domain_cell_vec->data(i,j,k,2) = Point(i,j,k).z; + domain_cell_mag->data.resize( domain->nx, domain->ny, domain->nz ); + domain_cell_vec->data.resize( + { (size_t) domain->nx, (size_t) domain->ny, (size_t) domain->nz, 3 } ); + for ( int i = 0; i < domain->nx; i++ ) { + for ( int j = 0; j < domain->ny; j++ ) { + for ( int k = 0; k < domain->nz; k++ ) { + domain_cell_mag->data( i, j, k ) = distance( Point( i, j, k ) ); + domain_cell_vec->data( i, j, k, 0 ) = Point( i, j, k ).x; + domain_cell_vec->data( i, j, k, 1 ) = Point( i, j, k ).y; + domain_cell_vec->data( i, j, k, 2 ) = Point( i, j, k ).z; } } } // Create the MeshDataStruct - std::vector meshData(4); + std::vector meshData( 4 ); meshData[0].meshName = "pointmesh"; - meshData[0].mesh = set1; - meshData[0].vars.push_back(point_node_mag); - meshData[0].vars.push_back(point_node_vec); + meshData[0].mesh = set1; + meshData[0].vars.push_back( point_node_mag ); + meshData[0].vars.push_back( point_node_vec ); meshData[1].meshName = "trimesh"; - meshData[1].mesh = trimesh; - meshData[1].vars.push_back(set_node_mag); - meshData[1].vars.push_back(set_node_vec); - meshData[1].vars.push_back(set_cell_mag); - meshData[1].vars.push_back(set_cell_vec); + meshData[1].mesh = trimesh; + meshData[1].vars.push_back( set_node_mag ); + meshData[1].vars.push_back( set_node_vec ); + meshData[1].vars.push_back( set_cell_mag ); + meshData[1].vars.push_back( set_cell_vec ); meshData[2].meshName = "trilist"; - meshData[2].mesh = trilist; - meshData[2].vars.push_back(list_node_mag); - meshData[2].vars.push_back(list_node_vec); - meshData[2].vars.push_back(list_cell_mag); - meshData[2].vars.push_back(list_cell_vec); + meshData[2].mesh = trilist; + meshData[2].vars.push_back( list_node_mag ); + meshData[2].vars.push_back( list_node_vec ); + meshData[2].vars.push_back( list_cell_mag ); + meshData[2].vars.push_back( list_cell_vec ); meshData[3].meshName = "domain"; - meshData[3].mesh = domain; - meshData[3].vars.push_back(domain_node_mag); - meshData[3].vars.push_back(domain_node_vec); - meshData[3].vars.push_back(domain_cell_mag); - meshData[3].vars.push_back(domain_cell_vec); + meshData[3].mesh = domain; + meshData[3].vars.push_back( domain_node_mag ); + meshData[3].vars.push_back( domain_node_vec ); + meshData[3].vars.push_back( domain_cell_mag ); + meshData[3].vars.push_back( domain_cell_vec ); + for ( const auto &data : meshData ) + ASSERT( data.check( true ) ); // Run the tests testWriter( "old", meshData, ut ); @@ -393,11 +401,9 @@ int main(int argc, char **argv) // Finished ut.report(); - PROFILE_SAVE("TestWriter",true); + PROFILE_SAVE( "TestWriter", true ); int N_errors = ut.NumFailGlobal(); comm.barrier(); Utilities::shutdown(); return N_errors; } - - From c1cd959da61968df006756b5ca30bff5b3bd5179 Mon Sep 17 00:00:00 2001 From: James McClure Date: Thu, 18 Mar 2021 14:51:26 -0400 Subject: [PATCH 20/32] D3Q7 Lee model (looks anisotropic) --- common/ScaLBL.cpp | 40 +++-- common/ScaLBL.h | 15 +- cpu/FreeLee.cpp | 339 +++++++++++++++++++++++++--------------- models/FreeLeeModel.cpp | 74 +++++++-- 4 files changed, 300 insertions(+), 168 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index dcadb08e..4726bae6 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -516,9 +516,9 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis n = k*Nx*Ny+j*Nx+i; if (id[n] > 0){ // Counts for the six faces - if (i>0 && i<=width) Map(n)=idx++; - else if (j>0 && j<=width) Map(n)=idx++; - else if (k>0 && k<=width) Map(n)=idx++; + if (i>0 && i<=width) Map(n)=idx++; + else if (j>0 && j<=width) Map(n)=idx++; + else if (k>0 && k<=width) Map(n)=idx++; else if (i>Nx-width-2 && iNy-width-2 && jNz-width-2 && k +#include #define STOKES @@ -70,6 +71,8 @@ extern "C" void ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init(double *gqbar, double gqbar[17*Np+n] = 0.0277777777777778*(p-0.5*(Fy-Fz)); ; //double(100*n)+17.f; gqbar[18*Np+n] = 0.0277777777777778*(p-0.5*(-Fy+Fz));; //double(100*n)+18.f; } + + } extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, double *Den, double *hq, double *ColorGrad, @@ -101,7 +104,8 @@ extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, doubl nz = nz/ColorMag_temp; theta = M*cs2_inv*(1-4.0*phi*phi)/W; - + theta = 0; // try more diffusive initial condition + hq[0*Np+idx]=0.3333333333333333*(phi); hq[1*Np+idx]=0.1111111111111111*(phi+theta*nx); hq[2*Np+idx]=0.1111111111111111*(phi-theta*nx); @@ -116,7 +120,7 @@ extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, doubl extern "C" void ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double rhoA, double rhoB, int start, int finish, int Np){ - int idx,n,nread; + int idx,nread; double fq,phi; for (int n=start; nSendD3Q7AA(hq,0); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(NeighborList, dvcMap, hq, Den, Phi, rhoA, rhoB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(NeighborList, dvcMap, hq, Den, Phi, ColorGrad, Velocity, rhoA, rhoB, tauM, W, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(hq,0); //WRITE INTO OPPOSITE ScaLBL_Comm->Barrier(); - ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(NeighborList, dvcMap, hq, Den, Phi, rhoA, rhoB, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(NeighborList, dvcMap, hq, Den, Phi, ColorGrad, Velocity, rhoA, rhoB, tauM, W, 0, ScaLBL_Comm->LastExterior(), Np); // Perform the collision operation - ScaLBL_Comm->SendD3Q19AA(gqbar); //READ FROM NORMAL + // Halo exchange for phase field + ScaLBL_D3Q7_ComputePhaseField(dvcMap, hq, Den, Phi, rhoA, rhoB, 0, ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_WideHalo->Send(Phi); + ScaLBL_Comm_WideHalo->Recv(Phi); if (BoundaryCondition > 0 && BoundaryCondition < 5){ //TODO to be revised + // Need to add BC for hq!!! ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); } - // Halo exchange for phase field - ScaLBL_Comm_WideHalo->Send(Phi); + printf("write debug strideY=%i strideZ = %i \n",Nxh, Nxh*Nyh); + WriteDebug_TwoFluid(); + + ScaLBL_Comm->SendD3Q19AA(gqbar); //READ FROM NORMAL ScaLBL_D3Q19_AAodd_FreeLeeModel(NeighborList, dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM, kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_WideHalo->Recv(Phi); + ScaLBL_Comm->RecvD3Q19AA(gqbar); //WRITE INTO OPPOSITE ScaLBL_Comm->Barrier(); // Set BCs @@ -793,33 +799,34 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){ ScaLBL_Comm->D3Q19_Reflection_BC_z(gqbar); ScaLBL_Comm->D3Q19_Reflection_BC_Z(gqbar); } + ScaLBL_D3Q19_AAodd_FreeLeeModel(NeighborList, dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM, kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->Barrier(); - printf("write debug strideY=%i strideZ = %i \n",Nxh, Nxh*Nyh); - WriteDebug_TwoFluid(); - + // *************EVEN TIMESTEP************* timestep++; // Compute the Phase indicator field ScaLBL_Comm->SendD3Q7AA(hq,0); //READ FROM NORMA - ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(dvcMap, hq, Den, Phi, rhoA, rhoB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAeven_FreeLee_PhaseField(dvcMap, hq, Den, Phi, ColorGrad, Velocity, rhoA, rhoB, tauM, W, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(hq,0); //WRITE INTO OPPOSITE ScaLBL_Comm->Barrier(); - ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(dvcMap, hq, Den, Phi, rhoA, rhoB, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_AAeven_FreeLee_PhaseField(dvcMap, hq, Den, Phi, ColorGrad, Velocity, rhoA, rhoB, tauM, W, 0, ScaLBL_Comm->LastExterior(), Np); // Perform the collision operation - ScaLBL_Comm->SendD3Q19AA(gqbar); //READ FORM NORMAL // Halo exchange for phase field + ScaLBL_D3Q7_ComputePhaseField(dvcMap, hq, Den, Phi, rhoA, rhoB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_WideHalo->Send(Phi); + ScaLBL_Comm_WideHalo->Recv(Phi); if (BoundaryCondition > 0 && BoundaryCondition < 5){ ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); } - ScaLBL_Comm_WideHalo->Send(Phi); + ScaLBL_Comm->SendD3Q19AA(gqbar); //READ FORM NORMAL + ScaLBL_D3Q19_AAeven_FreeLeeModel(dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM, kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_WideHalo->Recv(Phi); ScaLBL_Comm->RecvD3Q19AA(gqbar); //WRITE INTO OPPOSITE ScaLBL_Comm->Barrier(); // Set boundary conditions @@ -964,6 +971,32 @@ void ScaLBL_FreeLeeModel::WriteDebug_TwoFluid(){ DoubleArray PhaseData(Nxh,Nyh,Nzh); //ScaLBL_Comm->RegularLayout(Map,Phi,PhaseField); ScaLBL_CopyToHost(PhaseData.data(), Phi, sizeof(double)*Nh); + /* + IntArray MapData(Np); + ScaLBL_CopyToHost(MapData.data(), dvcMap, sizeof(int)*Np); + FILE *MAP; + sprintf(LocalRankFilename,"Map.%05i.raw",rank); + MAP = fopen(LocalRankFilename,"wb"); + fwrite(MapData.data(),4,Np,MAP); + fclose(MAP); + + FILE *NB; + //IntArray Neighbors(18,Np); + //ScaLBL_CopyToHost(Neighbors.data(), NeighborList, sizeof(int)*Np*18); + sprintf(LocalRankFilename,"neighbors.%05i.raw",rank); + NB = fopen(LocalRankFilename,"wb"); + fwrite(NeighborList,4,18*Np,NB); + fclose(NB); + + FILE *DIST; + DoubleArray DistData(7, Np); + ScaLBL_CopyToHost(DistData.data(), hq, 7*sizeof(double)*Np); + sprintf(LocalRankFilename,"h.%05i.raw",rank); + DIST = fopen(LocalRankFilename,"wb"); + fwrite(DistData.data(),8,7*Np,DIST); + fclose(DIST); + + */ FILE *OUTFILE; sprintf(LocalRankFilename,"Phase.%05i.raw",rank); @@ -972,6 +1005,17 @@ void ScaLBL_FreeLeeModel::WriteDebug_TwoFluid(){ fclose(OUTFILE); DoubleArray PhaseField(Nx,Ny,Nz); + FILE *DIST; + for (int q=0; q<7; q++){ + ScaLBL_Comm->RegularLayout(Map,&hq[q*Np],PhaseField); + + sprintf(LocalRankFilename,"h%i.%05i.raw",q,rank); + DIST = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,Nx*Ny*Nz,DIST); + fclose(DIST); + + } + ScaLBL_Comm->RegularLayout(Map,Den,PhaseField); FILE *AFILE; sprintf(LocalRankFilename,"Density.%05i.raw",rank); @@ -1082,7 +1126,7 @@ void ScaLBL_FreeLeeModel::Create_DummyPhase_MGTest(){ if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); Map.resize(Nx,Ny,Nz); Map.fill(-2); auto neighborList= new int[18*Npad]; - Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id.data(),Np,2); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id.data(),Np,1); comm.barrier(); //........................................................................... From eb62dbd38ecc9b6680ad72038b52555573992b74 Mon Sep 17 00:00:00 2001 From: James McClure Date: Thu, 18 Mar 2021 21:06:15 -0400 Subject: [PATCH 21/32] standalone d3q7 mass collision for color model --- common/ScaLBL.h | 6 ++ cpu/Color.cpp | 194 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 198 insertions(+), 2 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 89965f4e..4eb6dd7a 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -155,6 +155,12 @@ extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *NeighborList, int *Map, double extern "C" void ScaLBL_D3Q7_AAeven_PhaseField(int *Map, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q7_AAodd_Color(int *neighborList, int *Map, double *Aq, double *Bq, double *Den, + double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np); + +extern "C" void ScaLBL_D3Q7_AAeven_Color(int *Map, double *Aq, double *Bq, double *Den, + double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np); + extern "C" void ScaLBL_D3Q19_Gradient(int *Map, double *Phi, double *ColorGrad, int start, int finish, int Np, int Nx, int Ny, int Nz); extern "C" void ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz); diff --git a/cpu/Color.cpp b/cpu/Color.cpp index 35cbd5fd..6f67a6fc 100644 --- a/cpu/Color.cpp +++ b/cpu/Color.cpp @@ -2489,10 +2489,200 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di } } + +extern "C" void ScaLBL_D3Q7_AAodd_Color(int *neighborList, int *Map, double *Aq, double *Bq, double *Den, + double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np){ + + int nr1,nr2,nr3,nr4,nr5,nr6; + double nA,nB; // number density + double a1,b1,a2,b2,nAB,delta; + double C,nx,ny,nz; //color gradient magnitude and direction + double ux,uy,uz; + double phi; + // Instantiate mass transport distributions + // Stationary value - distribution 0 + for (int n=start; n0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; + + // q = 1 + //nread = neighborList[n+Np]; + Aq[nr2] = a1; + Bq[nr2] = b1; + // q=2 + //nread = neighborList[n]; + Aq[nr1] = a2; + Bq[nr1] = b2; + + //............................................... + // Cq = {0,1,0} + delta = beta*nA*nB*nAB*0.1111111111111111*ny; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; + + // q = 3 + //nread = neighborList[n+3*Np]; + Aq[nr4] = a1; + Bq[nr4] = b1; + // q = 4 + //nread = neighborList[n+2*Np]; + Aq[nr3] = a2; + Bq[nr3] = b2; + + //............................................... + // q = 4 + // Cq = {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nz; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; + + // q = 5 + //nread = neighborList[n+5*Np]; + Aq[nr6] = a1; + Bq[nr6] = b1; + // q = 6 + //nread = neighborList[n+4*Np]; + Aq[nr5] = a2; + Bq[nr5] = b2; + //............................................... + } +} + +extern "C" void ScaLBL_D3Q7_AAeven_Color(int *Map, double *Aq, double *Bq, double *Den, + double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np){ + + double nA,nB; // number density + double a1,b1,a2,b2,nAB,delta; + double C,nx,ny,nz; //color gradient magnitude and direction + double ux,uy,uz; + double phi; + // Instantiate mass transport distributions + // Stationary value - distribution 0 + for (int n=start; n0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; + + Aq[1*Np+n] = a1; + Bq[1*Np+n] = b1; + Aq[2*Np+n] = a2; + Bq[2*Np+n] = b2; + + //............................................... + // q = 2 + // Cq = {0,1,0} + delta = beta*nA*nB*nAB*0.1111111111111111*ny; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; + + Aq[3*Np+n] = a1; + Bq[3*Np+n] = b1; + Aq[4*Np+n] = a2; + Bq[4*Np+n] = b2; + //............................................... + // q = 4 + // Cq = {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nz; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; + + Aq[5*Np+n] = a1; + Bq[5*Np+n] = b1; + Aq[6*Np+n] = a2; + Bq[6*Np+n] = b2; + //............................................... + + } +} + + extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *neighborList, int *Map, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ - int idx,n,nread; + int idx,nread; double fq,nA,nB; for (int n=start; n Date: Fri, 19 Mar 2021 20:54:29 -0400 Subject: [PATCH 22/32] working bubble for FreeLee --- cpu/FreeLee.cpp | 27 +++++++++++++++------------ models/FreeLeeModel.cpp | 4 +--- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/cpu/FreeLee.cpp b/cpu/FreeLee.cpp index e32b4ca4..be9b903b 100644 --- a/cpu/FreeLee.cpp +++ b/cpu/FreeLee.cpp @@ -223,6 +223,7 @@ extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map double ux,uy,uz; double phi; double M = 2.0/9.0*(tauM-0.5);//diffusivity (or mobility) for the phase field D3Q7 + double factor = 1.0; for (int n=start; nColor_BC_z(dvcMap, Phi, Den, inletA, inletB); ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); } - - printf("write debug strideY=%i strideZ = %i \n",Nxh, Nxh*Nyh); - WriteDebug_TwoFluid(); ScaLBL_Comm->SendD3Q19AA(gqbar); //READ FROM NORMAL ScaLBL_D3Q19_AAodd_FreeLeeModel(NeighborList, dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM, @@ -865,6 +862,7 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){ if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); if (rank==0) printf("********************************************************\n"); + WriteDebug_TwoFluid(); // ************************************************************************ } From dadc9709474ac25f9cb37e08f570b427d9bd820d Mon Sep 17 00:00:00 2001 From: James McClure Date: Sat, 20 Mar 2021 13:12:49 -0400 Subject: [PATCH 23/32] clean up arguments for Free Lee --- common/ScaLBL.h | 8 +- cpu/FreeLee.cpp | 32 +- cuda/FreeLee.cu | 1406 ++++++++++++++++++++++++----------------------- 3 files changed, 734 insertions(+), 712 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 4eb6dd7a..3bc752dc 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -206,12 +206,12 @@ extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, dou extern "C" void ScaLBL_D3Q7_ComputePhaseField(int *Map, double *hq, double *Den, double *Phi, double rhoA, double rhoB, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, double *dist, double *hq, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, - double rhoA, double rhoB, double tauA, double tauB, double tauM, double kappa, double beta, double W, double Fx, double Fy, double Fz, +extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, double *dist, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, + double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double *hq, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, - double rhoA, double rhoB, double tauA, double tauB, double tauM, double kappa, double beta, double W, double Fx, double Fy, double Fz, +extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, + double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborList, double *dist, double *Vel, double *Pressure, diff --git a/cpu/FreeLee.cpp b/cpu/FreeLee.cpp index be9b903b..16a9b3bd 100644 --- a/cpu/FreeLee.cpp +++ b/cpu/FreeLee.cpp @@ -244,7 +244,7 @@ extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map //Normalize the Color Gradient C = sqrt(nx*nx+ny*ny+nz*nz); double ColorMag = C; - if (C < 1.0e-8) ColorMag=1.0; + if (C < 1.0e-12) ColorMag=1.0; nx = nx/ColorMag; ny = ny/ColorMag; nz = nz/ColorMag; @@ -292,15 +292,7 @@ extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map hq[nr6] = h5; hq[nr5] = h6; //........................................................................ - - //phi = h0+h1+h2+h3+h4+h5+h6; - - // save the number densities - //Den[n] = rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); - - // save the phase indicator field - //Phi[idx] = phi; - + } } @@ -335,7 +327,7 @@ extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, dou //Normalize the Color Gradient C = sqrt(nx*nx+ny*ny+nz*nz); double ColorMag = C; - if (C < 1.0e-8) ColorMag=1.0; + if (C < 1.0e-12) ColorMag=1.0; nx = nx/ColorMag; ny = ny/ColorMag; nz = nz/ColorMag; @@ -368,15 +360,7 @@ extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, dou hq[5*Np+n] = h5; hq[6*Np+n] = h6; //........................................................................ - - //phi = h0+h1+h2+h3+h4+h5+h6; - - // save the number densities - //Den[n] = rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); - - // save the phase indicator field - //Phi[idx] = phi; - + } } @@ -408,8 +392,8 @@ extern "C" void ScaLBL_D3Q7_ComputePhaseField(int *Map, double *hq, double *Den } -extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, double *dist, double *hq, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, - double rhoA, double rhoB, double tauA, double tauB, double tauM, double kappa, double beta, double W, double Fx, double Fy, double Fz, +extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, double *dist, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, + double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ int nn,nn2x,ijk; @@ -960,8 +944,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, dou } } -extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double *hq, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, - double rhoA, double rhoB, double tauA, double tauB, double tauM, double kappa, double beta, double W, double Fx, double Fy, double Fz, +extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, + double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ int nn,nn2x,ijk; diff --git a/cuda/FreeLee.cu b/cuda/FreeLee.cu index e37a92a3..3d9afccb 100644 --- a/cuda/FreeLee.cu +++ b/cuda/FreeLee.cu @@ -185,10 +185,16 @@ __global__ void dvc_ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(int *neighborList, } } -__global__ void dvc_ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(int *Map, double *hq, double *Den, double *Phi, - double rhoA, double rhoB, int start, int finish, int Np){ - int idx,n; - double fq,phi; +__global__ void dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel, + double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){ + + int idx,nr1,nr2,nr3,nr4,nr5,nr6; + double h0,h1,h2,h3,h4,h5,h6; + double nx,ny,nz,C; + double ux,uy,uz; + double phi; + double M = 2.0/9.0*(tauM-0.5);//diffusivity (or mobility) for the phase field D3Q7 + double factor = 1.0; // for (int n=start; n Date: Sat, 20 Mar 2021 13:13:00 -0400 Subject: [PATCH 24/32] clean up arguments for Free Lee --- models/FreeLeeModel.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/models/FreeLeeModel.cpp b/models/FreeLeeModel.cpp index 209a36c9..a2c7dcda 100644 --- a/models/FreeLeeModel.cpp +++ b/models/FreeLeeModel.cpp @@ -778,7 +778,7 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){ } ScaLBL_Comm->SendD3Q19AA(gqbar); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_FreeLeeModel(NeighborList, dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM, + ScaLBL_D3Q19_AAodd_FreeLeeModel(NeighborList, dvcMap, gqbar, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q19AA(gqbar); //WRITE INTO OPPOSITE @@ -797,7 +797,7 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){ ScaLBL_Comm->D3Q19_Reflection_BC_Z(gqbar); } - ScaLBL_D3Q19_AAodd_FreeLeeModel(NeighborList, dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM, + ScaLBL_D3Q19_AAodd_FreeLeeModel(NeighborList, dvcMap, gqbar, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->Barrier(); @@ -822,7 +822,7 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){ } ScaLBL_Comm->SendD3Q19AA(gqbar); //READ FORM NORMAL - ScaLBL_D3Q19_AAeven_FreeLeeModel(dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM, + ScaLBL_D3Q19_AAeven_FreeLeeModel(dvcMap, gqbar, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q19AA(gqbar); //WRITE INTO OPPOSITE ScaLBL_Comm->Barrier(); @@ -839,7 +839,7 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){ ScaLBL_Comm->D3Q19_Reflection_BC_z(gqbar); ScaLBL_Comm->D3Q19_Reflection_BC_Z(gqbar); } - ScaLBL_D3Q19_AAeven_FreeLeeModel(dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM, + ScaLBL_D3Q19_AAeven_FreeLeeModel(dvcMap, gqbar, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->Barrier(); //************************************************************************ From 68b881bdb03d964417a86dbda6db67bed3b62157 Mon Sep 17 00:00:00 2001 From: James McClure Date: Sat, 20 Mar 2021 18:36:09 -0400 Subject: [PATCH 25/32] draft for cuda version of free lee model --- cuda/FreeLee.cu | 78 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/cuda/FreeLee.cu b/cuda/FreeLee.cu index 3d9afccb..7352585a 100644 --- a/cuda/FreeLee.cu +++ b/cuda/FreeLee.cu @@ -1,4 +1,5 @@ #include +#include #define STOKES @@ -2001,54 +2002,119 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dis } extern "C" void ScaLBL_D3Q19_FreeLeeModel_TwoFluid_Init(double *gqbar, double *mu_phi, double *ColorGrad, double Fx, double Fy, double Fz, int Np){ - + + dvc_ScaLBL_D3Q19_FreeLeeModel_TwoFluid_Init<<>>( gqbar, mu_phi, ColorGrad, Fx, Fy, Fz, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_FreeLeeModel_TwoFluid_Init: %s \n",cudaGetErrorString(err)); + } } extern "C" void ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init(double *gqbar, double Fx, double Fy, double Fz, int Np){ - + + ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init<<>>( gqbar, Fx, Fy, Fz, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init: %s \n",cudaGetErrorString(err)); + } } extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, double *Den, double *hq, double *ColorGrad, double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){ + ScaLBL_FreeLeeModel_PhaseField_Init<<>>(Map, Phi, Den, hq, ColorGrad, rhoA, rhoB, tauM, W, start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_FreeLeeModel_PhaseField_Init: %s \n",cudaGetErrorString(err)); + } + + } extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np) { - /* need to add launcher */ + cudaFuncSetCacheConfig(ScaLBL_D3Q7_AAodd_FreeLee_PhaseField, cudaFuncCachePreferL1); + ScaLBL_D3Q7_AAodd_FreeLee_PhaseField<<>>(neighborList, Map, hq, Den, Phi, ColorGrad, Vel, + rhoA, rhoB, tauM, W, start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAodd_FreeLee_PhaseField: %s \n",cudaGetErrorString(err)); + } } extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){ - /****/ + + cudaFuncSetCacheConfig(ScaLBL_D3Q7_AAeven_FreeLee_PhaseField, cudaFuncCachePreferL1); + ScaLBL_D3Q7_AAeven_FreeLee_PhaseField<<>>( Map, hq, Den, Phi, ColorGrad, Vel, rhoA, rhoB, tauM, W, start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAeven_FreeLee_PhaseField: %s \n",cudaGetErrorString(err)); + } } extern "C" void ScaLBL_D3Q7_ComputePhaseField(int *Map, double *hq, double *Den, double *Phi, double rhoA, double rhoB, int start, int finish, int Np){ + cudaFuncSetCacheConfig(ScaLBL_D3Q7_ComputePhaseField, cudaFuncCachePreferL1); + ScaLBL_D3Q7_ComputePhaseField<<>>( Map, hq, Den, Phi, rhoA, rhoB, start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_ComputePhaseField: %s \n",cudaGetErrorString(err)); + } + } extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, double *dist, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ + + cudaFuncSetCacheConfig(ScaLBL_D3Q19_AAodd_FreeLeeModel, cudaFuncCachePreferL1); + ScaLBL_D3Q19_AAodd_FreeLeeModel<<>>(neighborList, Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad, + rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_FreeLeeModel: %s \n",cudaGetErrorString(err)); + } } extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ + cudaFuncSetCacheConfig(ScaLBL_D3Q19_AAeven_FreeLeeModel, cudaFuncCachePreferL1); + ScaLBL_D3Q19_AAeven_FreeLeeModel<<>>(Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad, + rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_FreeLeeModel: %s \n",cudaGetErrorString(err)); + } + } extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborList, double *dist, double *Vel, double *Pressure, double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){ - + cudaFuncSetCacheConfig(ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK, cudaFuncCachePreferL1); + ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK<<>>(neighborList, dist, Vel, Pressure, + tau, Fx, Fy, Fz, start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK: %s \n",cudaGetErrorString(err)); + } } extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, double *Vel, double *Pressure, double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){ - + + cudaFuncSetCacheConfig(ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK, cudaFuncCachePreferL1); + ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK<<>>(dist, Vel, Pressure, + tau, Fx, Fy, Fz, start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK: %s \n",cudaGetErrorString(err)); + } } From f3b7f6ee7e953f998acceb18160ac967298fdc65 Mon Sep 17 00:00:00 2001 From: James McClure Date: Sat, 20 Mar 2021 20:39:33 -0400 Subject: [PATCH 26/32] fix ubuntu config --- sample_scripts/configure_ubuntu | 1 - 1 file changed, 1 deletion(-) diff --git a/sample_scripts/configure_ubuntu b/sample_scripts/configure_ubuntu index fab18662..290e8078 100755 --- a/sample_scripts/configure_ubuntu +++ b/sample_scripts/configure_ubuntu @@ -6,7 +6,6 @@ cmake -D CMAKE_C_COMPILER:PATH=/opt/arden/openmpi/3.1.2/bin/mpicc \ -D CMAKE_CXX_FLAGS="-O3 -fPIC " \ -D CMAKE_CXX_STANDARD=14 \ -D MPIEXEC=mpirun \ - -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \ -D CMAKE_BUILD_TYPE:STRING=Release \ -D CUDA_FLAGS="-arch sm_35" \ -D CUDA_HOST_COMPILER="/usr/bin/gcc" \ From 9f9b0dbffe45868e6ec23493b89a610836bb3b6d Mon Sep 17 00:00:00 2001 From: James McClure Date: Sat, 20 Mar 2021 22:24:28 -0400 Subject: [PATCH 27/32] adding functions to get data for analysis routines --- models/FreeLeeModel.cpp | 39 +++++++++++++++++++++++++++++++++++++++ models/FreeLeeModel.h | 6 +++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/models/FreeLeeModel.cpp b/models/FreeLeeModel.cpp index a2c7dcda..ef6f0772 100644 --- a/models/FreeLeeModel.cpp +++ b/models/FreeLeeModel.cpp @@ -20,6 +20,45 @@ Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0), ScaLBL_FreeLeeModel::~ScaLBL_FreeLeeModel(){ } + + +void ScaLBL_FreeLeeModel::getPhase(DoubleArray &PhaseValues){ + + DoubleArray PhaseWideHalo(Nxh,Nyh,Nzh); + ScaLBL_CopyToHost(PhaseWideHalo.data(), Phi, sizeof(double)*Nh); + + // use halo width = 1 for analysis data + for (int k=1; kRegularLayout(Map,Pressure,PressureValues); + ScaLBL_Comm->Barrier(); comm.barrier(); + + ScaLBL_Comm->RegularLayout(Map,mu_phi,MuValues); + ScaLBL_Comm->Barrier(); comm.barrier(); + +} + +void ScaLBL_FreeLeeModel::getVelocity(DoubleArray &Vel_x, DoubleArray &Vel_y, DoubleArray &Vel_z){ + + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Vel_x); + ScaLBL_Comm->Barrier(); comm.barrier(); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Vel_y); + ScaLBL_Comm->Barrier(); comm.barrier(); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Vel_z); + ScaLBL_Comm->Barrier(); comm.barrier(); +} + void ScaLBL_FreeLeeModel::ReadParams(string filename){ // read the input database db = std::make_shared( filename ); diff --git a/models/FreeLeeModel.h b/models/FreeLeeModel.h index 23afe39b..af1906db 100644 --- a/models/FreeLeeModel.h +++ b/models/FreeLeeModel.h @@ -76,8 +76,12 @@ public: double *Velocity; double *Pressure; + void getPhase(DoubleArray &PhaseValues); + void getPotential(DoubleArray &PressureValues, DoubleArray &MuValues); + void getVelocity(DoubleArray &Vx, DoubleArray &Vy, DoubleArray &Vz); + DoubleArray SignDist; - + private: Utilities::MPI comm; From d913b9bdc8c091537c06cdd083b376cab864e330 Mon Sep 17 00:00:00 2001 From: James McClure Date: Sun, 21 Mar 2021 00:15:23 -0400 Subject: [PATCH 28/32] added visualization capability for Lee model --- analysis/FreeEnergy.cpp | 181 +++++++++++++++++++++++++++++++ analysis/FreeEnergy.h | 54 +++++++++ models/FreeLeeModel.cpp | 28 ++--- models/FreeLeeModel.h | 9 +- tests/lbpm_freelee_simulator.cpp | 30 ++++- tests/testGlobalMassFreeLee.cpp | 2 +- 6 files changed, 278 insertions(+), 26 deletions(-) create mode 100644 analysis/FreeEnergy.cpp create mode 100644 analysis/FreeEnergy.h diff --git a/analysis/FreeEnergy.cpp b/analysis/FreeEnergy.cpp new file mode 100644 index 00000000..6a641a95 --- /dev/null +++ b/analysis/FreeEnergy.cpp @@ -0,0 +1,181 @@ +#include "analysis/FreeEnergy.h" + +FreeEnergyAnalyzer::FreeEnergyAnalyzer(std::shared_ptr dm): + Dm(dm) +{ + + Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz; + Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0; + + ChemicalPotential.resize(Nx,Ny,Nz); ChemicalPotential.fill(0); + Phi.resize(Nx,Ny,Nz); Phi.fill(0); + Pressure.resize(Nx,Ny,Nz); Pressure.fill(0); + Rho.resize(Nx,Ny,Nz); Rho.fill(0); + Vel_x.resize(Nx,Ny,Nz); Vel_x.fill(0); // Gradient of the phase indicator field + Vel_y.resize(Nx,Ny,Nz); Vel_y.fill(0); + Vel_z.resize(Nx,Ny,Nz); Vel_z.fill(0); + SDs.resize(Nx,Ny,Nz); SDs.fill(0); + + if (Dm->rank()==0){ + bool WriteHeader=false; + TIMELOG = fopen("free.csv","r"); + if (TIMELOG != NULL) + fclose(TIMELOG); + else + WriteHeader=true; + + TIMELOG = fopen("free.csv","a+"); + if (WriteHeader) + { + // If timelog is empty, write a short header to list the averages + //fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n"); + fprintf(TIMELOG,"timestep\n"); + } + } + +} + +FreeEnergyAnalyzer::~FreeEnergyAnalyzer(){ + if (Dm->rank()==0){ + fclose(TIMELOG); + } +} + +void FreeEnergyAnalyzer::SetParams(){ + +} + +void FreeEnergyAnalyzer::Basic(ScaLBL_FreeLeeModel &LeeModel, int timestep){ + + int i,j,k; + + if (Dm->rank()==0){ + fprintf(TIMELOG,"%i ",timestep); + /*for (int ion=0; ion input_db, int timestep){ + + auto vis_db = input_db->getDatabase( "Visualization" ); + char VisName[40]; + + std::vector visData; + fillHalo fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1); + + IO::initialize("","silo","false"); + // Create the MeshDataStruct + visData.resize(1); + + visData[0].meshName = "domain"; + visData[0].mesh = std::make_shared( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz ); + auto VisPhase = std::make_shared(); + auto VisPressure = std::make_shared(); + auto VisChemicalPotential = std::make_shared(); + auto VxVar = std::make_shared(); + auto VyVar = std::make_shared(); + auto VzVar = std::make_shared(); + + + if (vis_db->getWithDefault( "save_phase_field", true )){ + VisPhase->name = "Phase"; + VisPhase->type = IO::VariableType::VolumeVariable; + VisPhase->dim = 1; + VisPhase->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VisPhase); + } + + if (vis_db->getWithDefault( "save_potential", true )){ + + VisPressure->name = "Pressure"; + VisPressure->type = IO::VariableType::VolumeVariable; + VisPressure->dim = 1; + VisPressure->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VisPressure); + + VisChemicalPotential->name = "ChemicalPotential"; + VisChemicalPotential->type = IO::VariableType::VolumeVariable; + VisChemicalPotential->dim = 1; + VisChemicalPotential->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VisChemicalPotential); + } + + if (vis_db->getWithDefault( "save_velocity", false )){ + VxVar->name = "Velocity_x"; + VxVar->type = IO::VariableType::VolumeVariable; + VxVar->dim = 1; + VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VxVar); + VyVar->name = "Velocity_y"; + VyVar->type = IO::VariableType::VolumeVariable; + VyVar->dim = 1; + VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VyVar); + VzVar->name = "Velocity_z"; + VzVar->type = IO::VariableType::VolumeVariable; + VzVar->dim = 1; + VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VzVar); + } + + if (vis_db->getWithDefault( "save_phase", true )){ + ASSERT(visData[0].vars[0]->name=="Phase"); + LeeModel.getPhase(Phi); + Array& PhaseData = visData[0].vars[0]->data; + fillData.copy(Phi,PhaseData); + } + + if (vis_db->getWithDefault( "save_potential", true )){ + ASSERT(visData[0].vars[1]->name=="Pressure"); + LeeModel.getPotential(Pressure, ChemicalPotential); + Array& PressureData = visData[0].vars[1]->data; + fillData.copy(Pressure,PressureData); + + ASSERT(visData[0].vars[2]->name=="ChemicalPotential"); + Array& ChemicalPotentialData = visData[0].vars[2]->data; + fillData.copy(ChemicalPotential,ChemicalPotentialData); + } + + if (vis_db->getWithDefault( "save_velocity", false )){ + ASSERT(visData[0].vars[3]->name=="Velocity_x"); + ASSERT(visData[0].vars[4]->name=="Velocity_y"); + ASSERT(visData[0].vars[5]->name=="Velocity_z"); + LeeModel.getVelocity(Vel_x,Vel_y,Vel_z); + Array& VelxData = visData[0].vars[3]->data; + Array& VelyData = visData[0].vars[4]->data; + Array& VelzData = visData[0].vars[5]->data; + fillData.copy(Vel_x,VelxData); + fillData.copy(Vel_y,VelyData); + fillData.copy(Vel_z,VelzData); + } + + if (vis_db->getWithDefault( "write_silo", true )) + IO::writeData( timestep, visData, Dm->Comm ); + +/* if (vis_db->getWithDefault( "save_8bit_raw", true )){ + char CurrentIDFilename[40]; + sprintf(CurrentIDFilename,"id_t%d.raw",timestep); + Averages.AggregateLabels(CurrentIDFilename); + } +*/ +} diff --git a/analysis/FreeEnergy.h b/analysis/FreeEnergy.h new file mode 100644 index 00000000..fbb1ba31 --- /dev/null +++ b/analysis/FreeEnergy.h @@ -0,0 +1,54 @@ +/* + * averaging tools for electrochemistry + */ + +#ifndef FreeEnergyAnalyzer_INC +#define FreeEnergyAnalyzer_INC + +#include +#include "common/Domain.h" +#include "common/Utilities.h" +#include "common/MPI.h" +#include "common/Communication.h" +#include "analysis/analysis.h" +#include "analysis/distance.h" +#include "analysis/Minkowski.h" +#include "analysis/SubPhase.h" +#include "IO/MeshDatabase.h" +#include "IO/Reader.h" +#include "IO/Writer.h" +#include "models/FreeLeeModel.h" + +class FreeEnergyAnalyzer{ +public: + std::shared_ptr Dm; + double Volume; + // input variables + double rho_n, rho_w; + double nu_n, nu_w; + double gamma_wn, beta; + double Fx, Fy, Fz; + + //........................................................................... + int Nx,Ny,Nz; + DoubleArray Rho; + DoubleArray Phi; + DoubleArray ChemicalPotential; + DoubleArray Pressure; + DoubleArray Vel_x; + DoubleArray Vel_y; + DoubleArray Vel_z; + DoubleArray SDs; + + FreeEnergyAnalyzer(std::shared_ptr Dm); + ~FreeEnergyAnalyzer(); + + void SetParams(); + void Basic( ScaLBL_FreeLeeModel &LeeModel, int timestep); + void WriteVis( ScaLBL_FreeLeeModel &LeeModel, std::shared_ptr input_db, int timestep); + +private: + FILE *TIMELOG; +}; +#endif + diff --git a/models/FreeLeeModel.cpp b/models/FreeLeeModel.cpp index ef6f0772..428db40f 100644 --- a/models/FreeLeeModel.cpp +++ b/models/FreeLeeModel.cpp @@ -31,7 +31,7 @@ void ScaLBL_FreeLeeModel::getPhase(DoubleArray &PhaseValues){ for (int k=1; k( t2 - t1 ).count() / timestep; + double cputime = std::chrono::duration( t2 - t1 ).count() / (EXIT_TIME-START_TIME); // Performance obtained from each node double MLUPS = double(Np)/cputime/1000000; - if (rank==0) printf("********************************************************\n"); - if (rank==0) printf("CPU time = %f \n", cputime); - if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); - MLUPS *= nprocs; - if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); - if (rank==0) printf("********************************************************\n"); - - WriteDebug_TwoFluid(); - // ************************************************************************ + return MLUPS; } void ScaLBL_FreeLeeModel::Run_SingleFluid(){ diff --git a/models/FreeLeeModel.h b/models/FreeLeeModel.h index af1906db..17cc6323 100644 --- a/models/FreeLeeModel.h +++ b/models/FreeLeeModel.h @@ -16,6 +16,9 @@ Implementation of Lee et al JCP 2016 lattice boltzmann model #include "common/ScaLBL.h" #include "common/WideHalo.h" +#ifndef ScaLBL_FreeLeeModel_INC +#define ScaLBL_FreeLeeModel_INC + class ScaLBL_FreeLeeModel{ public: ScaLBL_FreeLeeModel(int RANK, int NP, const Utilities::MPI& COMM); @@ -28,11 +31,13 @@ public: void ReadInput(); void Create_TwoFluid(); void Initialize_TwoFluid(); - void Run_TwoFluid(); + double Run_TwoFluid(int returntime); + void WriteDebug_TwoFluid(); void Create_SingleFluid(); void Initialize_SingleFluid(); void Run_SingleFluid(); + void WriteDebug_SingleFluid(); // test utilities void Create_DummyPhase_MGTest(); @@ -97,4 +102,4 @@ private: void AssignComponentLabels_ChemPotential_ColorGrad(); }; - +#endif diff --git a/tests/lbpm_freelee_simulator.cpp b/tests/lbpm_freelee_simulator.cpp index 3663c4e9..0508c43a 100644 --- a/tests/lbpm_freelee_simulator.cpp +++ b/tests/lbpm_freelee_simulator.cpp @@ -8,6 +8,7 @@ #include "common/Utilities.h" #include "models/FreeLeeModel.h" +#include "analysis/FreeEnergy.h" //******************************************************************* // Implementation of Free-Energy Two-Phase LBM (Lee model) @@ -52,10 +53,33 @@ int main( int argc, char **argv ) LeeModel.SetDomain(); LeeModel.ReadInput(); LeeModel.Create_TwoFluid(); + + FreeEnergyAnalyzer Analysis(LeeModel.Dm); + LeeModel.Initialize_TwoFluid(); - LeeModel.Run_TwoFluid(); - LeeModel.WriteDebug_TwoFluid(); - + + /*** RUN MAIN TIMESTEPS HERE ************/ + double MLUPS=0.0; + int timestep = 0; + int visualization_time = LeeModel.timestepMax; + if (LeeModel.vis_db->keyExists( "visualizataion_interval" )){ + visualization_time = LeeModel.vis_db->getScalar( "visualizataion_interval" ); + timestep += visualization_time; + } + while (LeeModel.timestep < LeeModel.timestepMax){ + MLUPS = LeeModel.Run_TwoFluid(timestep); + if (rank==0) printf("Lattice update rate (per MPI process)= %f MLUPS \n", MLUPS); + Analysis.WriteVis(LeeModel,LeeModel.db, timestep); + timestep += visualization_time; + } + //LeeModel.WriteDebug_TwoFluid(); + if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + MLUPS *= nprocs; + if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); + if (rank==0) printf("********************************************************\n"); + // ************************************************************************ + PROFILE_STOP("Main"); auto file = db->getWithDefault( "TimerFile", "lbpm_freelee_simulator" ); auto level = db->getWithDefault( "TimerLevel", 1 ); diff --git a/tests/testGlobalMassFreeLee.cpp b/tests/testGlobalMassFreeLee.cpp index c9073b9a..2e976854 100644 --- a/tests/testGlobalMassFreeLee.cpp +++ b/tests/testGlobalMassFreeLee.cpp @@ -63,7 +63,7 @@ int main( int argc, char **argv ) DoubleArray DensityInit(Nx,Ny,Nz); LeeModel.ScaLBL_Comm->RegularLayout(LeeModel.Map,LeeModel.Den,DensityInit); - LeeModel.Run_TwoFluid(); + double MLUPS = LeeModel.Run_TwoFluid(LeeModel.timestepMax); DoubleArray DensityFinal(Nx,Ny,Nz); LeeModel.ScaLBL_Comm->RegularLayout(LeeModel.Map,LeeModel.Den,DensityFinal); From 32d750a04cfd7b75bca83ca79f9c25571836a8e1 Mon Sep 17 00:00:00 2001 From: James McClure Date: Mon, 22 Mar 2021 10:45:30 -0400 Subject: [PATCH 29/32] GPU build for lee model --- cuda/FreeLee.cu | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/cuda/FreeLee.cu b/cuda/FreeLee.cu index 7352585a..45bbf65b 100644 --- a/cuda/FreeLee.cu +++ b/cuda/FreeLee.cu @@ -1,4 +1,5 @@ #include +#include #include #define STOKES @@ -189,7 +190,7 @@ __global__ void dvc_ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(int *neighborList, __global__ void dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){ - int idx,nr1,nr2,nr3,nr4,nr5,nr6; + int n,idx,nr1,nr2,nr3,nr4,nr5,nr6; double h0,h1,h2,h3,h4,h5,h6; double nx,ny,nz,C; double ux,uy,uz; @@ -380,7 +381,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ - int nn,nn2x,ijk; + int n,nn,nn2x,ijk; int nr1,nr2,nr3,nr4,nr5,nr6,nr7,nr8,nr9,nr10,nr11,nr12,nr13,nr14,nr15,nr16,nr17,nr18; double ux,uy,uz;//fluid velocity double p;//pressure @@ -936,8 +937,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, dou double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ - int nn,nn2x,ijk; - //int nr1,nr2,nr3,nr4,nr5,nr6,nr7,nr8,nr9,nr10,nr11,nr12,nr13,nr14,nr15,nr16,nr17,nr18; + int n,nn,nn2x,ijk; double ux,uy,uz;//fluid velocity double p;//pressure double chem;//chemical potential @@ -2013,7 +2013,7 @@ extern "C" void ScaLBL_D3Q19_FreeLeeModel_TwoFluid_Init(double *gqbar, double *m extern "C" void ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init(double *gqbar, double Fx, double Fy, double Fz, int Np){ - ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init<<>>( gqbar, Fx, Fy, Fz, Np); + dvc_ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init<<>>( gqbar, Fx, Fy, Fz, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init: %s \n",cudaGetErrorString(err)); @@ -2023,7 +2023,7 @@ extern "C" void ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init(double *gqbar, double extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, double *Den, double *hq, double *ColorGrad, double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){ - ScaLBL_FreeLeeModel_PhaseField_Init<<>>(Map, Phi, Den, hq, ColorGrad, rhoA, rhoB, tauM, W, start, finish, Np); + dvc_ScaLBL_FreeLeeModel_PhaseField_Init<<>>(Map, Phi, Den, hq, ColorGrad, rhoA, rhoB, tauM, W, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_FreeLeeModel_PhaseField_Init: %s \n",cudaGetErrorString(err)); @@ -2034,8 +2034,8 @@ extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, doubl extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np) { - cudaFuncSetCacheConfig(ScaLBL_D3Q7_AAodd_FreeLee_PhaseField, cudaFuncCachePreferL1); - ScaLBL_D3Q7_AAodd_FreeLee_PhaseField<<>>(neighborList, Map, hq, Den, Phi, ColorGrad, Vel, + cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField, cudaFuncCachePreferL1); + dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField<<>>(neighborList, Map, hq, Den, Phi, ColorGrad, Vel, rhoA, rhoB, tauM, W, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -2046,8 +2046,8 @@ extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){ - cudaFuncSetCacheConfig(ScaLBL_D3Q7_AAeven_FreeLee_PhaseField, cudaFuncCachePreferL1); - ScaLBL_D3Q7_AAeven_FreeLee_PhaseField<<>>( Map, hq, Den, Phi, ColorGrad, Vel, rhoA, rhoB, tauM, W, start, finish, Np); + cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField, cudaFuncCachePreferL1); + dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField<<>>( Map, hq, Den, Phi, ColorGrad, Vel, rhoA, rhoB, tauM, W, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_D3Q7_AAeven_FreeLee_PhaseField: %s \n",cudaGetErrorString(err)); @@ -2057,13 +2057,12 @@ extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, dou extern "C" void ScaLBL_D3Q7_ComputePhaseField(int *Map, double *hq, double *Den, double *Phi, double rhoA, double rhoB, int start, int finish, int Np){ - cudaFuncSetCacheConfig(ScaLBL_D3Q7_ComputePhaseField, cudaFuncCachePreferL1); - ScaLBL_D3Q7_ComputePhaseField<<>>( Map, hq, Den, Phi, rhoA, rhoB, start, finish, Np); + cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q7_ComputePhaseField, cudaFuncCachePreferL1); + dvc_ScaLBL_D3Q7_ComputePhaseField<<>>( Map, hq, Den, Phi, rhoA, rhoB, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_D3Q7_ComputePhaseField: %s \n",cudaGetErrorString(err)); } - } @@ -2071,8 +2070,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, dou double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ - cudaFuncSetCacheConfig(ScaLBL_D3Q19_AAodd_FreeLeeModel, cudaFuncCachePreferL1); - ScaLBL_D3Q19_AAodd_FreeLeeModel<<>>(neighborList, Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad, + cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel, cudaFuncCachePreferL1); + dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel<<>>(neighborList, Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -2084,8 +2083,8 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ - cudaFuncSetCacheConfig(ScaLBL_D3Q19_AAeven_FreeLeeModel, cudaFuncCachePreferL1); - ScaLBL_D3Q19_AAeven_FreeLeeModel<<>>(Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad, + cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel, cudaFuncCachePreferL1); + dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel<<>>(Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -2097,9 +2096,9 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborList, double *dist, double *Vel, double *Pressure, double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){ - cudaFuncSetCacheConfig(ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK, cudaFuncCachePreferL1); - ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK<<>>(neighborList, dist, Vel, Pressure, - tau, Fx, Fy, Fz, start, finish, Np); + cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK, cudaFuncCachePreferL1); + dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK<<>>(neighborList, dist, Vel, Pressure, + tau, rho0, Fx, Fy, Fz, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK: %s \n",cudaGetErrorString(err)); @@ -2109,12 +2108,15 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborLis extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, double *Vel, double *Pressure, double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){ - cudaFuncSetCacheConfig(ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK, cudaFuncCachePreferL1); - ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK<<>>(dist, Vel, Pressure, - tau, Fx, Fy, Fz, start, finish, Np); + cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK, cudaFuncCachePreferL1); + dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK<<>>(dist, Vel, Pressure, + tau, rho0, Fx, Fy, Fz, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK: %s \n",cudaGetErrorString(err)); } } + +extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad,int strideY, int strideZ, int start, int finish, int Np){ +} \ No newline at end of file From f44167d2ef8ef7d449389c25d7741509feb1f50f Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 24 Mar 2021 07:57:27 -0400 Subject: [PATCH 30/32] add hip version of Lee model --- hip/FreeLee.cu | 1483 ++++++++++++++++++++++++++---------------------- 1 file changed, 794 insertions(+), 689 deletions(-) diff --git a/hip/FreeLee.cu b/hip/FreeLee.cu index 558bd2f1..09bc8689 100644 --- a/hip/FreeLee.cu +++ b/hip/FreeLee.cu @@ -1,11 +1,12 @@ #include +#include #include "hip/hip_runtime.h" +#define STOKES + #define NBLOCKS 1024 #define NTHREADS 256 -#define STOKES - __global__ void dvc_ScaLBL_D3Q19_FreeLeeModel_TwoFluid_Init(double *gqbar, double *mu_phi, double *ColorGrad, double Fx, double Fy, double Fz, int Np) { int n; @@ -186,10 +187,16 @@ __global__ void dvc_ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(int *neighborList, } } -__global__ void dvc_ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(int *Map, double *hq, double *Den, double *Phi, - double rhoA, double rhoB, int start, int finish, int Np){ - int idx,n; - double fq,phi; +__global__ void dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel, + double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){ + + int n,idx,nr1,nr2,nr3,nr4,nr5,nr6; + double h0,h1,h2,h3,h4,h5,h6; + double nx,ny,nz,C; + double ux,uy,uz; + double phi; + double M = 2.0/9.0*(tauM-0.5);//diffusivity (or mobility) for the phase field D3Q7 + double factor = 1.0; // for (int n=start; n>>( gqbar, mu_phi, ColorGrad, Fx, Fy, Fz, Np); + hipError_t err = hipGetLastError(); + if (hipSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_FreeLeeModel_TwoFluid_Init: %s \n",hipGetErrorString(err)); + } } extern "C" void ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init(double *gqbar, double Fx, double Fy, double Fz, int Np){ - + + dvc_ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init<<>>( gqbar, Fx, Fy, Fz, Np); + hipError_t err = hipGetLastError(); + if (hipSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init: %s \n",hipGetErrorString(err)); + } } extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, double *Den, double *hq, double *ColorGrad, double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){ -} -extern "C" void ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, - double rhoA, double rhoB, int start, int finish, int Np){ + dvc_ScaLBL_FreeLeeModel_PhaseField_Init<<>>(Map, Phi, Den, hq, ColorGrad, rhoA, rhoB, tauM, W, start, finish, Np); + hipError_t err = hipGetLastError(); + if (hipSuccess != err){ + printf("CUDA error in ScaLBL_FreeLeeModel_PhaseField_Init: %s \n",hipGetErrorString(err)); + } + } - -extern "C" void ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(int *Map, double *hq, double *Den, double *Phi, - double rhoA, double rhoB, int start, int finish, int Np){ - +extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel, + double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np) +{ + hipFuncSetCacheConfig(dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField, hipFuncCachePreferL1); + dvc_ScaLBL_D3Q7_AAodd_FreeLee_PhaseField<<>>(neighborList, Map, hq, Den, Phi, ColorGrad, Vel, + rhoA, rhoB, tauM, W, start, finish, Np); + hipError_t err = hipGetLastError(); + if (hipSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAodd_FreeLee_PhaseField: %s \n",hipGetErrorString(err)); + } } -extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, double *dist, double *hq, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, - double rhoA, double rhoB, double tauA, double tauB, double tauM, double kappa, double beta, double W, double Fx, double Fy, double Fz, +extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel, + double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){ + + hipFuncSetCacheConfig(dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField, hipFuncCachePreferL1); + dvc_ScaLBL_D3Q7_AAeven_FreeLee_PhaseField<<>>( Map, hq, Den, Phi, ColorGrad, Vel, rhoA, rhoB, tauM, W, start, finish, Np); + hipError_t err = hipGetLastError(); + if (hipSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAeven_FreeLee_PhaseField: %s \n",hipGetErrorString(err)); + } +} + + +extern "C" void ScaLBL_D3Q7_ComputePhaseField(int *Map, double *hq, double *Den, double *Phi, double rhoA, double rhoB, int start, int finish, int Np){ + + hipFuncSetCacheConfig(dvc_ScaLBL_D3Q7_ComputePhaseField, hipFuncCachePreferL1); + dvc_ScaLBL_D3Q7_ComputePhaseField<<>>( Map, hq, Den, Phi, rhoA, rhoB, start, finish, Np); + hipError_t err = hipGetLastError(); + if (hipSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_ComputePhaseField: %s \n",hipGetErrorString(err)); + } +} + + +extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, double *dist, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, + double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ -} - + hipFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel, hipFuncCachePreferL1); + dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel<<>>(neighborList, Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad, + rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); + hipError_t err = hipGetLastError(); + if (hipSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_FreeLeeModel: %s \n",hipGetErrorString(err)); + } +} -extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double *hq, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, - double rhoA, double rhoB, double tauA, double tauB, double tauM, double kappa, double beta, double W, double Fx, double Fy, double Fz, +extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, + double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ + hipFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel, hipFuncCachePreferL1); + dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel<<>>(Map, dist, Den, Phi, mu_phi, Vel, Pressure, ColorGrad, + rhoA, rhoB, tauA, tauB, kappa, beta, W, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); + hipError_t err = hipGetLastError(); + if (hipSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_FreeLeeModel: %s \n",hipGetErrorString(err)); + } + } extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborList, double *dist, double *Vel, double *Pressure, double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){ - + hipFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK, hipFuncCachePreferL1); + dvc_ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK<<>>(neighborList, dist, Vel, Pressure, + tau, rho0, Fx, Fy, Fz, start, finish, Np); + hipError_t err = hipGetLastError(); + if (hipSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK: %s \n",hipGetErrorString(err)); + } } extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, double *Vel, double *Pressure, double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){ - + + hipFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK, hipFuncCachePreferL1); + dvc_ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK<<>>(dist, Vel, Pressure, + tau, rho0, Fx, Fy, Fz, start, finish, Np); + hipError_t err = hipGetLastError(); + if (hipSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK: %s \n",hipGetErrorString(err)); + } } + +extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad,int strideY, int strideZ, int start, int finish, int Np){ +} \ No newline at end of file From aba4ca8455576b277776fd8b708af30757a08872 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 24 Mar 2021 21:18:50 -0400 Subject: [PATCH 31/32] add extra check on wall displacement --- analysis/morphology.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/analysis/morphology.cpp b/analysis/morphology.cpp index 37f58d0c..f21767dd 100644 --- a/analysis/morphology.cpp +++ b/analysis/morphology.cpp @@ -702,12 +702,14 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array &id, if (rank == 0) printf(" delta=%f, growth=%f, max. displacement = %f \n",morph_delta, GrowthEstimate, MAX_DISPLACEMENT); // Now adjust morph_delta - double step_size = (TargetGrowth - GrowthEstimate)*(morph_delta - morph_delta_previous) / (GrowthEstimate - GrowthPrevious); - GrowthPrevious = GrowthEstimate; - morph_delta_previous = morph_delta; - morph_delta += step_size; + if (fabs(GrowthEstimate - GrowthPrevious) > 0.0) { + double step_size = (TargetGrowth - GrowthEstimate)*(morph_delta - morph_delta_previous) / (GrowthEstimate - GrowthPrevious); + GrowthPrevious = GrowthEstimate; + morph_delta_previous = morph_delta; + morph_delta += step_size; + } if (morph_delta / morph_delta_previous > 2.0 ) morph_delta = morph_delta_previous*2.0; - + //MAX_DISPLACEMENT *= max(TargetGrowth/GrowthEstimate,1.25); if (morph_delta > 0.0 ){ From 1ddf5e709e3fd7de732d5323c62e0769ecda1aea Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 24 Mar 2021 21:19:53 -0400 Subject: [PATCH 32/32] refactor analysis to take color model as argument --- analysis/runAnalysis.cpp | 133 +++++++++++++++++++++++++++++++ analysis/runAnalysis.h | 3 + models/ColorModel.cpp | 126 ++++++++++++++++++++++++++++- models/ColorModel.h | 6 ++ tests/lbpm_color_simulator.cpp | 36 +++++++-- tests/lbpm_freelee_simulator.cpp | 4 +- 6 files changed, 297 insertions(+), 11 deletions(-) diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index f43a26ff..ab40ae4c 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -706,6 +706,139 @@ runAnalysis::runAnalysis( std::shared_ptr input_db, const RankInfoStru } + // Initialize the comms + for ( int i = 0; i < 1024; i++ ) + d_comm_used[i] = false; + // Initialize the threads + int N_threads = db->getWithDefault( "N_threads", 4 ); + auto method = db->getWithDefault( "load_balance", "default" ); + createThreads( method, N_threads ); +} + +runAnalysis::runAnalysis( ScaLBL_ColorModel &ColorModel) +/* std::shared_ptr input_db, const RankInfoStruct &rank_info, + std::shared_ptr ScaLBL_Comm, std::shared_ptr Dm, int Np, + bool Regular, IntArray Map ) + : d_Np( Np ), + d_regular( Regular ), + d_rank_info( rank_info ), + d_Map( Map ), + d_comm( Dm->Comm.dup() ), + d_ScaLBL_Comm( ScaLBL_Comm )*/ +{ + + d_comm = ColorModel.Dm->Comm.dup(); + d_Np = ColorModel.Np; + bool Regular = false; + + auto input_db = ColorModel.db; + auto db = input_db->getDatabase( "Analysis" ); + auto vis_db = input_db->getDatabase( "Visualization" ); + + // Ids of work items to use for dependencies + ThreadPool::thread_id_t d_wait_blobID; + ThreadPool::thread_id_t d_wait_analysis; + ThreadPool::thread_id_t d_wait_vis; + ThreadPool::thread_id_t d_wait_restart; + ThreadPool::thread_id_t d_wait_subphase; + + char rankString[20]; + sprintf( rankString, "%05d", ColorModel.Dm->rank() ); + d_n[0] = ColorModel.Dm->Nx - 2; + d_n[1] = ColorModel.Dm->Ny - 2; + d_n[2] = ColorModel.Dm->Nz - 2; + d_N[0] = ColorModel.Dm->Nx; + d_N[1] = ColorModel.Dm->Ny; + d_N[2] = ColorModel.Dm->Nz; + + d_restart_interval = db->getScalar( "restart_interval" ); + d_analysis_interval = db->getScalar( "analysis_interval" ); + d_subphase_analysis_interval = INT_MAX; + d_visualization_interval = INT_MAX; + d_blobid_interval = INT_MAX; + if ( db->keyExists( "blobid_interval" ) ) { + d_blobid_interval = db->getScalar( "blobid_interval" ); + } + if ( db->keyExists( "visualization_interval" ) ) { + d_visualization_interval = db->getScalar( "visualization_interval" ); + } + if ( db->keyExists( "subphase_analysis_interval" ) ) { + d_subphase_analysis_interval = db->getScalar( "subphase_analysis_interval" ); + } + + auto restart_file = db->getScalar( "restart_file" ); + d_restartFile = restart_file + "." + rankString; + + + d_rank = d_comm.getRank(); + writeIDMap( ID_map_struct(), 0, id_map_filename ); + // Initialize IO for silo + IO::initialize( "", "silo", "false" ); + // Create the MeshDataStruct + d_meshData.resize( 1 ); + + d_meshData[0].meshName = "domain"; + d_meshData[0].mesh = std::make_shared( + d_rank_info, d_n[0], d_n[1], d_n[2], ColorModel.Dm->Lx, ColorModel.Dm->Ly, ColorModel.Dm->Lz ); + auto PhaseVar = std::make_shared(); + auto PressVar = std::make_shared(); + auto VxVar = std::make_shared(); + auto VyVar = std::make_shared(); + auto VzVar = std::make_shared(); + auto SignDistVar = std::make_shared(); + auto BlobIDVar = std::make_shared(); + + if ( vis_db->getWithDefault( "save_phase_field", true ) ) { + PhaseVar->name = "phase"; + PhaseVar->type = IO::VariableType::VolumeVariable; + PhaseVar->dim = 1; + PhaseVar->data.resize( d_n[0], d_n[1], d_n[2] ); + d_meshData[0].vars.push_back( PhaseVar ); + } + + if ( vis_db->getWithDefault( "save_pressure", false ) ) { + PressVar->name = "Pressure"; + PressVar->type = IO::VariableType::VolumeVariable; + PressVar->dim = 1; + PressVar->data.resize( d_n[0], d_n[1], d_n[2] ); + d_meshData[0].vars.push_back( PressVar ); + } + + if ( vis_db->getWithDefault( "save_velocity", false ) ) { + VxVar->name = "Velocity_x"; + VxVar->type = IO::VariableType::VolumeVariable; + VxVar->dim = 1; + VxVar->data.resize( d_n[0], d_n[1], d_n[2] ); + d_meshData[0].vars.push_back( VxVar ); + VyVar->name = "Velocity_y"; + VyVar->type = IO::VariableType::VolumeVariable; + VyVar->dim = 1; + VyVar->data.resize( d_n[0], d_n[1], d_n[2] ); + d_meshData[0].vars.push_back( VyVar ); + VzVar->name = "Velocity_z"; + VzVar->type = IO::VariableType::VolumeVariable; + VzVar->dim = 1; + VzVar->data.resize( d_n[0], d_n[1], d_n[2] ); + d_meshData[0].vars.push_back( VzVar ); + } + + if ( vis_db->getWithDefault( "save_distance", false ) ) { + SignDistVar->name = "SignDist"; + SignDistVar->type = IO::VariableType::VolumeVariable; + SignDistVar->dim = 1; + SignDistVar->data.resize( d_n[0], d_n[1], d_n[2] ); + d_meshData[0].vars.push_back( SignDistVar ); + } + + if ( vis_db->getWithDefault( "save_connected_components", false ) ) { + BlobIDVar->name = "BlobID"; + BlobIDVar->type = IO::VariableType::VolumeVariable; + BlobIDVar->dim = 1; + BlobIDVar->data.resize( d_n[0], d_n[1], d_n[2] ); + d_meshData[0].vars.push_back( BlobIDVar ); + } + + // Initialize the comms for ( int i = 0; i < 1024; i++ ) d_comm_used[i] = false; diff --git a/analysis/runAnalysis.h b/analysis/runAnalysis.h index a82c4ba0..c7c4ce71 100644 --- a/analysis/runAnalysis.h +++ b/analysis/runAnalysis.h @@ -7,6 +7,7 @@ #include "common/Communication.h" #include "common/ScaLBL.h" #include "threadpool/thread_pool.h" +#include "models/ColorModel.h" #include @@ -31,6 +32,8 @@ public: runAnalysis( std::shared_ptr db, const RankInfoStruct &rank_info, std::shared_ptr ScaLBL_Comm, std::shared_ptr dm, int Np, bool Regular, IntArray Map ); + + runAnalysis( ScaLBL_ColorModel &ColorModel); //! Destructor ~runAnalysis(); diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 1fe00824..204fd1d6 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -531,6 +531,121 @@ void ScaLBL_ColorModel::Initialize(){ ScaLBL_CopyToHost(Averages->Phi.data(),Phi,N*sizeof(double)); } +double ScaLBL_ColorModel::Run(int returntime){ + int nprocs=nprocx*nprocy*nprocz; + + //************ MAIN ITERATION LOOP ***************************************/ + comm.barrier(); + PROFILE_START("Loop"); + //std::shared_ptr analysis_db; + bool Regular = false; + auto current_db = db->cloneDatabase(); + auto t1 = std::chrono::system_clock::now(); + int START_TIMESTEP = timestep; + int EXIT_TIMESTEP = min(timestepMax,returntime); + while (timestep < EXIT_TIMESTEP ) { + //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } + PROFILE_START("Update"); + // *************ODD TIMESTEP************* + timestep++; + // Compute the Phase indicator field + // Read for Aq, Bq happens in this routine (requires communication) + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_Comm->Barrier(); + ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); + + // Perform the collision operation + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL + if (BoundaryCondition > 0 && BoundaryCondition < 5){ + ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); + ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); + } + // Halo exchange for phase field + ScaLBL_Comm_Regular->SendHalo(Phi); + + ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(Phi); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_Comm->Barrier(); + // Set BCs + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + if (BoundaryCondition == 4){ + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 5){ + ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); + ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); + } + ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->Barrier(); + + // *************EVEN TIMESTEP************* + timestep++; + // Compute the Phase indicator field + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_Comm->Barrier(); + ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); + + // Perform the collision operation + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL + // Halo exchange for phase field + if (BoundaryCondition > 0 && BoundaryCondition < 5){ + ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); + ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); + } + ScaLBL_Comm_Regular->SendHalo(Phi); + ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(Phi); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_Comm->Barrier(); + // Set boundary conditions + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 4){ + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 5){ + ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); + ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); + } + ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->Barrier(); + //************************************************************************ + } + PROFILE_STOP("Update"); + + PROFILE_STOP("Loop"); + PROFILE_SAVE("lbpm_color_simulator",1); + //************************************************************************ + // Compute the walltime per timestep + auto t2 = std::chrono::system_clock::now(); + double cputime = std::chrono::duration( t2 - t1 ).count() / (timestep - START_TIMESTEP); + // Performance obtained from each node + double MLUPS = double(Np)/cputime/1000000; + + if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("CPU time = %f \n", cputime); + if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + return(MLUPS); + MLUPS *= nprocs; + +} + void ScaLBL_ColorModel::Run(){ int nprocs=nprocx*nprocy*nprocz; const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); @@ -580,7 +695,6 @@ void ScaLBL_ColorModel::Run(){ if (color_db->keyExists( "krA_morph_factor" )){ KRA_MORPH_FACTOR = color_db->getScalar( "krA_morph_factor" ); } - /* defaults for simulation protocols */ auto protocol = color_db->getWithDefault( "protocol", "none" ); if (protocol == "image sequence"){ @@ -625,7 +739,7 @@ void ScaLBL_ColorModel::Run(){ if (analysis_db->keyExists( "seed_water" )){ seed_water = analysis_db->getScalar( "seed_water" ); if (rank == 0) printf("Seed water in oil %f (seed_water) \n",seed_water); - USE_SEED = true; + ASSERT(protocol == "seed water"); } if (analysis_db->keyExists( "morph_delta" )){ morph_delta = analysis_db->getScalar( "morph_delta" ); @@ -656,7 +770,6 @@ void ScaLBL_ColorModel::Run(){ MAX_MORPH_TIMESTEPS = analysis_db->getScalar( "max_morph_timesteps" ); } - if (rank==0){ printf("********************************************************\n"); if (protocol == "image sequence"){ @@ -1320,7 +1433,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta double vF = 0.f; double vS = 0.f; double delta_volume; - double WallFactor = 0.0; + double WallFactor = 1.0; bool USE_CONNECTED_NWP = false; DoubleArray phase(Nx,Ny,Nz); @@ -1343,6 +1456,11 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta } } double volume_initial = Dm->Comm.sumReduce( count); + double PoreVolume = Dm->Volume*Dm->Porosity(); + /*ensure target isn't an absurdly small fraction of pore volume */ + if (volume_initial < target_delta_volume*PoreVolume){ + volume_initial = target_delta_volume*PoreVolume; + } /* sprintf(LocalRankFilename,"phi_initial.%05i.raw",rank); FILE *INPUT = fopen(LocalRankFilename,"wb"); diff --git a/models/ColorModel.h b/models/ColorModel.h index b2a9c1d1..7d3c858a 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -16,6 +16,10 @@ Implementation of color lattice boltzmann model #include "ProfilerApp.h" #include "threadpool/thread_pool.h" + +#ifndef ScaLBL_ColorModel_INC +#define ScaLBL_ColorModel_INC + class ScaLBL_ColorModel{ public: ScaLBL_ColorModel(int RANK, int NP, const Utilities::MPI& COMM); @@ -29,6 +33,7 @@ public: void Create(); void Initialize(); void Run(); + double Run(int returntime); void WriteDebug(); void getPhaseField(DoubleArray &f); @@ -99,4 +104,5 @@ private: int timestep; int timestep_previous; }; +#endif diff --git a/tests/lbpm_color_simulator.cpp b/tests/lbpm_color_simulator.cpp index 590d5b8e..d62bef0f 100644 --- a/tests/lbpm_color_simulator.cpp +++ b/tests/lbpm_color_simulator.cpp @@ -27,19 +27,24 @@ int main( int argc, char **argv ) // Initialize Utilities::startup( argc, argv ); - // Load the input database - auto db = std::make_shared( argv[1] ); - { // Limit scope so variables that contain communicators will free before MPI_Finialize Utilities::MPI comm( MPI_COMM_WORLD ); int rank = comm.getRank(); int nprocs = comm.getSize(); + std::string SimulationMode = "production"; + // Load the input database + auto db = std::make_shared( argv[1] ); + if (argc > 2) { + SimulationMode = "development"; + } if ( rank == 0 ) { printf( "********************************************************\n" ); printf( "Running Color LBM \n" ); printf( "********************************************************\n" ); + if (SimulationMode == "development") + printf("**** DEVELOPMENT MODE ENABLED *************\n"); } // Initialize compute device int device = ScaLBL_SetDevice( rank ); @@ -62,8 +67,29 @@ int main( int argc, char **argv ) ColorModel.Create(); // creating the model will create data structure to match the pore // structure and allocate variables ColorModel.Initialize(); // initializing the model will set initial conditions for variables - ColorModel.Run(); - // ColorModel.WriteDebug(); + + if (SimulationMode == "development"){ + double MLUPS=0.0; + int timestep = 0; + int analysis_interval = ColorModel.timestepMax; + if (ColorModel.analysis_db->keyExists( "" )){ + analysis_interval = ColorModel.analysis_db->getScalar( "analysis_interval" ); + } + FlowAdaptor Adapt(ColorModel); + runAnalysis analysis(ColorModel); + while (ColorModel.timestep < ColorModel.timestepMax){ + timestep += analysis_interval; + MLUPS = ColorModel.Run(timestep); + if (rank==0) printf("Lattice update rate (per MPI process)= %f MLUPS \n", MLUPS); + + Adapt.MoveInterface(ColorModel); + } + } //Analysis.WriteVis(LeeModel,LeeModel.db, timestep); + + else + ColorModel.Run(); + + ColorModel.WriteDebug(); PROFILE_STOP( "Main" ); auto file = db->getWithDefault( "TimerFile", "lbpm_color_simulator" ); diff --git a/tests/lbpm_freelee_simulator.cpp b/tests/lbpm_freelee_simulator.cpp index 0508c43a..0f003baa 100644 --- a/tests/lbpm_freelee_simulator.cpp +++ b/tests/lbpm_freelee_simulator.cpp @@ -62,8 +62,8 @@ int main( int argc, char **argv ) double MLUPS=0.0; int timestep = 0; int visualization_time = LeeModel.timestepMax; - if (LeeModel.vis_db->keyExists( "visualizataion_interval" )){ - visualization_time = LeeModel.vis_db->getScalar( "visualizataion_interval" ); + if (LeeModel.vis_db->keyExists( "visualization_interval" )){ + visualization_time = LeeModel.vis_db->getScalar( "visualization_interval" ); timestep += visualization_time; } while (LeeModel.timestep < LeeModel.timestepMax){