From 64b49f720e1534f2aba054da7062bfea0449f4b8 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 6 Jan 2021 11:58:43 -0500 Subject: [PATCH] merge complete / cpu tests --- IO/PIO.cpp | 2 +- IO/silo.cpp | 2 +- IO/silo.h | 2 +- IO/silo.hpp | 2 +- analysis/ElectroChemistry.h | 2 +- analysis/GreyPhase.h | 2 +- analysis/SubPhase.h | 2 +- common/SpherePack.cpp | 2 +- common/SpherePack.h | 2 +- models/DFHModel.h | 2 +- models/GreyscaleColorModel.h | 2 +- models/GreyscaleModel.h | 2 +- models/MultiPhysController.h | 2 +- tests/TestColorSquareTube.cpp | 2 +- tests/TestFluxBC.cpp | 2 +- tests/TestInterfaceSpeed.cpp | 11 ++- tests/TestIonModel.cpp | 23 +++-- tests/TestMassConservationD3Q7.cpp | 2 +- tests/TestMicroCTReader.cpp | 10 ++- tests/TestSetDevice.cpp | 20 ++--- tests/TestWriter.cpp | 12 ++- tests/lbpm_color_simulator.cpp | 23 ++--- ...m_electrokinetic_SingleFluid_simulator.cpp | 22 +++-- tests/lbpm_greyscaleColor_simulator.cpp | 29 +++---- tests/lbpm_greyscale_simulator.cpp | 87 +++++++++---------- tests/lbpm_minkowski_scalar.cpp | 2 +- 26 files changed, 128 insertions(+), 143 deletions(-) diff --git a/IO/PIO.cpp b/IO/PIO.cpp index 6c6ece2d..fe0f7db4 100644 --- a/IO/PIO.cpp +++ b/IO/PIO.cpp @@ -1,6 +1,6 @@ #include "IO/PIO.h" #include "common/Utilities.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include #include diff --git a/IO/silo.cpp b/IO/silo.cpp index eece8583..ddf3646a 100644 --- a/IO/silo.cpp +++ b/IO/silo.cpp @@ -1,6 +1,6 @@ #include "IO/silo.h" #include "common/Utilities.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "ProfilerApp.h" diff --git a/IO/silo.h b/IO/silo.h index e200bb05..40a023d7 100644 --- a/IO/silo.h +++ b/IO/silo.h @@ -6,7 +6,7 @@ #include #include "common/Array.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "common/Communication.h" diff --git a/IO/silo.hpp b/IO/silo.hpp index 312f32d8..35852004 100644 --- a/IO/silo.hpp +++ b/IO/silo.hpp @@ -3,7 +3,7 @@ #include "IO/silo.h" #include "common/Utilities.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "ProfilerApp.h" diff --git a/analysis/ElectroChemistry.h b/analysis/ElectroChemistry.h index beaff833..90874ca0 100644 --- a/analysis/ElectroChemistry.h +++ b/analysis/ElectroChemistry.h @@ -8,7 +8,7 @@ #include #include "common/Domain.h" #include "common/Utilities.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "common/Communication.h" #include "analysis/analysis.h" #include "analysis/distance.h" diff --git a/analysis/GreyPhase.h b/analysis/GreyPhase.h index 4aca756d..3ab46752 100644 --- a/analysis/GreyPhase.h +++ b/analysis/GreyPhase.h @@ -10,7 +10,7 @@ #include "common/Communication.h" #include "analysis/analysis.h" #include "common/Utilities.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "IO/MeshDatabase.h" #include "IO/Reader.h" #include "IO/Writer.h" diff --git a/analysis/SubPhase.h b/analysis/SubPhase.h index 71b87ef0..691c654f 100644 --- a/analysis/SubPhase.h +++ b/analysis/SubPhase.h @@ -12,7 +12,7 @@ #include "analysis/distance.h" #include "analysis/Minkowski.h" #include "common/Utilities.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "IO/MeshDatabase.h" #include "IO/Reader.h" #include "IO/Writer.h" diff --git a/common/SpherePack.cpp b/common/SpherePack.cpp index a7246b72..3f77cefd 100644 --- a/common/SpherePack.cpp +++ b/common/SpherePack.cpp @@ -9,7 +9,7 @@ #include "common/Array.h" #include "common/Utilities.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "common/Communication.h" #include "common/Database.h" #include "common/SpherePack.h" diff --git a/common/SpherePack.h b/common/SpherePack.h index 5075b289..5f68dd7d 100644 --- a/common/SpherePack.h +++ b/common/SpherePack.h @@ -12,7 +12,7 @@ #include "common/Array.h" #include "common/Utilities.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "common/Communication.h" #include "common/Database.h" diff --git a/models/DFHModel.h b/models/DFHModel.h index b50f62a2..00e6e6b3 100644 --- a/models/DFHModel.h +++ b/models/DFHModel.h @@ -12,7 +12,7 @@ Implementation of color lattice boltzmann model #include "common/Communication.h" #include "analysis/TwoPhase.h" #include "analysis/runAnalysis.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "ProfilerApp.h" #include "threadpool/thread_pool.h" diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h index 1ae4ab73..667099e9 100644 --- a/models/GreyscaleColorModel.h +++ b/models/GreyscaleColorModel.h @@ -11,7 +11,7 @@ Implementation of two-fluid greyscale color lattice boltzmann model #include "common/Communication.h" #include "analysis/GreyPhase.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "ProfilerApp.h" #include "threadpool/thread_pool.h" diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h index 46cfb014..aa68c180 100644 --- a/models/GreyscaleModel.h +++ b/models/GreyscaleModel.h @@ -10,7 +10,7 @@ Implementation of color lattice boltzmann model #include #include "common/Communication.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "common/Database.h" #include "common/ScaLBL.h" #include "ProfilerApp.h" diff --git a/models/MultiPhysController.h b/models/MultiPhysController.h index dfc5bcee..4388d6b9 100644 --- a/models/MultiPhysController.h +++ b/models/MultiPhysController.h @@ -13,7 +13,7 @@ #include "common/ScaLBL.h" #include "common/Communication.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "analysis/Minkowski.h" #include "ProfilerApp.h" diff --git a/tests/TestColorSquareTube.cpp b/tests/TestColorSquareTube.cpp index e21aa286..1434c327 100644 --- a/tests/TestColorSquareTube.cpp +++ b/tests/TestColorSquareTube.cpp @@ -7,7 +7,7 @@ #include #include #include "common/ScaLBL.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "models/ColorModel.h" std::shared_ptr loadInputs( int nprocs ) diff --git a/tests/TestFluxBC.cpp b/tests/TestFluxBC.cpp index 3762aee6..3028d8ee 100644 --- a/tests/TestFluxBC.cpp +++ b/tests/TestFluxBC.cpp @@ -1,5 +1,5 @@ #include -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "common/Utilities.h" #include "common/ScaLBL.h" diff --git a/tests/TestInterfaceSpeed.cpp b/tests/TestInterfaceSpeed.cpp index 67bf8f95..4036a205 100644 --- a/tests/TestInterfaceSpeed.cpp +++ b/tests/TestInterfaceSpeed.cpp @@ -21,7 +21,8 @@ int main (int argc, char *argv[]) Utilities::startup( argc, argv ); Utilities::MPI comm( MPI_COMM_WORLD ); int rank = comm.getRank(); - + int toReturn = 0; + { // Load inputs string FILENAME = argv[1]; // Load inputs @@ -114,7 +115,6 @@ int main (int argc, char *argv[]) printf("-------------------------------- \n"); //......................................................................... - int toReturn = 0; if (fabs(Averages->awn - 2*PI*RADIUS*RADIUS)/(2*PI*RADIUS*RADIUS) > 0.02){ toReturn = 1; printf("TestCylinderArea.cpp: error tolerance exceeded for wn area \n"); @@ -144,9 +144,8 @@ int main (int argc, char *argv[]) toReturn = 7; } - return toReturn; - comm.barrier(); - return 0; - Utilities::shutdown(); + } + Utilities::shutdown(); + return toReturn; } diff --git a/tests/TestIonModel.cpp b/tests/TestIonModel.cpp index 2a0a02a9..58c051dc 100644 --- a/tests/TestIonModel.cpp +++ b/tests/TestIonModel.cpp @@ -23,21 +23,22 @@ int main(int argc, char **argv) Utilities::startup( argc, argv ); { // Limit scope so variables that contain communicators will free before MPI_Finialize - - MPI_Comm comm; - MPI_Comm_dup(MPI_COMM_WORLD,&comm); - int rank = comm_rank(comm); - int nprocs = comm_size(comm); + // Initialize MPI + Utilities::startup( argc, argv ); + Utilities::MPI comm( MPI_COMM_WORLD ); + int rank = comm.getRank(); + int nprocs = comm.getSize(); if (rank == 0){ printf("**************************************\n"); printf("Running Test for Ion Transport \n"); printf("**************************************\n"); } - // Initialize compute device - ScaLBL_SetDevice(rank); - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); + // Initialize compute device + int device=ScaLBL_SetDevice(rank); + NULL_USE( device ); + ScaLBL_DeviceBarrier(); + comm.barrier(); PROFILE_ENABLE(1); //PROFILE_ENABLE_TRACE(); @@ -62,7 +63,6 @@ int main(int argc, char **argv) IonModel.DummyFluidVelocity(); IonModel.DummyElectricField(); - int timestep=0; double error = 1.0; vectorci_avg_previous{0.0,0.0};//assuming 1:1 solution @@ -85,8 +85,7 @@ int main(int argc, char **argv) PROFILE_SAVE("TestIonModel",1); // **************************************************** - MPI_Barrier(comm); - MPI_Comm_free(&comm); + comm.barrier(); } // Limit scope so variables that contain communicators will free before MPI_Finialize diff --git a/tests/TestMassConservationD3Q7.cpp b/tests/TestMassConservationD3Q7.cpp index 3893781b..03b6d01a 100644 --- a/tests/TestMassConservationD3Q7.cpp +++ b/tests/TestMassConservationD3Q7.cpp @@ -8,7 +8,7 @@ #include #include "common/ScaLBL.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "models/ColorModel.h" inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius){ diff --git a/tests/TestMicroCTReader.cpp b/tests/TestMicroCTReader.cpp index 27230183..d8609356 100644 --- a/tests/TestMicroCTReader.cpp +++ b/tests/TestMicroCTReader.cpp @@ -1,6 +1,6 @@ // Test reading high-resolution files from the microct database -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "common/UnitTest.h" #include "common/Database.h" #include "common/Domain.h" @@ -13,12 +13,14 @@ void testReadMicroCT( const std::string& filename, UnitTest& ut ) { + Utilities::MPI comm( MPI_COMM_WORLD ); + // Get the domain info auto db = std::make_shared( filename ); auto domain_db = db->getDatabase( "Domain" ); // Test reading microCT files - auto data = readMicroCT( *domain_db, MPI_COMM_WORLD ); + auto data = readMicroCT( *domain_db, comm ); // Check if we loaded the data correctly if ( data.size() == domain_db->getVector( "n" ) ) @@ -30,7 +32,7 @@ void testReadMicroCT( const std::string& filename, UnitTest& ut ) auto n = domain_db->getVector( "n" ); auto nproc = domain_db->getVector( "nproc" ); int N[3] = { n[0]*nproc[0], n[1]*nproc[1], n[2]*nproc[2] }; - int rank = comm_rank(MPI_COMM_WORLD); + int rank = comm.getRank(); RankInfoStruct rankInfo( rank, nproc[0], nproc[1], nproc[2] ); std::vector meshData( 1 ); auto Var = std::make_shared(); @@ -41,7 +43,7 @@ void testReadMicroCT( const std::string& filename, UnitTest& ut ) meshData[0].meshName = "grid"; meshData[0].mesh = std::make_shared(rankInfo,n[0],n[1],n[2],N[0],N[1],N[2]); meshData[0].vars.push_back(Var); - IO::writeData( 0, meshData, MPI_COMM_WORLD ); + IO::writeData( 0, meshData, comm ); } diff --git a/tests/TestSetDevice.cpp b/tests/TestSetDevice.cpp index 51e71682..553f297d 100644 --- a/tests/TestSetDevice.cpp +++ b/tests/TestSetDevice.cpp @@ -1,24 +1,25 @@ #include -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "common/Utilities.h" #include "common/ScaLBL.h" int main (int argc, char **argv) -{ - MPI_Init(&argc,&argv); - int rank = MPI_WORLD_RANK(); - int nprocs = MPI_WORLD_SIZE(); +{ + Utilities::startup( argc, argv ); + Utilities::MPI comm( MPI_COMM_WORLD ); + int rank = comm.getRank(); + int nprocs = comm.getSize(); for (int i=0; i& meshData, UnitTest& ut ) { - int rank, nprocs; - MPI_Comm comm = MPI_COMM_WORLD; - MPI_Comm_rank(comm,&rank); - MPI_Comm_size(comm,&nprocs); - MPI_Barrier(comm); + Utilities::MPI comm( MPI_COMM_WORLD ); + int nprocs = comm.getSize(); + comm.barrier(); // Get the format std::string format2 = format; @@ -63,7 +61,7 @@ void testWriter( const std::string& format, std::vector& mes IO::initialize( "test_"+format, format2, false ); IO::writeData( 0, meshData, comm ); IO::writeData( 3, meshData, comm ); - MPI_Barrier(comm); + comm.barrier(); PROFILE_STOP(format+"-write"); // Get the summary name for reading diff --git a/tests/lbpm_color_simulator.cpp b/tests/lbpm_color_simulator.cpp index 996b7879..1d579486 100644 --- a/tests/lbpm_color_simulator.cpp +++ b/tests/lbpm_color_simulator.cpp @@ -23,6 +23,13 @@ int main(int argc, char **argv) { + + // Initialize MPI + Utilities::startup( argc, argv ); + Utilities::MPI comm( MPI_COMM_WORLD ); + int rank = comm.getRank(); + int nprocs = comm.getSize(); + // Load the input database auto db = std::make_shared( argv[1] ); @@ -33,20 +40,16 @@ int main(int argc, char **argv) { // Limit scope so variables that contain communicators will free before MPI_Finialize - MPI_Comm comm; - MPI_Comm_dup(MPI_COMM_WORLD,&comm); - int rank = comm_rank(comm); - int nprocs = comm_size(comm); - if (rank == 0){ printf("********************************************************\n"); printf("Running Color LBM \n"); printf("********************************************************\n"); } - // Initialize compute device - ScaLBL_SetDevice(rank); - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); + // Initialize compute device + int device=ScaLBL_SetDevice(rank); + NULL_USE( device ); + ScaLBL_DeviceBarrier(); + comm.barrier(); PROFILE_ENABLE(1); //PROFILE_ENABLE_TRACE(); @@ -71,8 +74,6 @@ int main(int argc, char **argv) PROFILE_SAVE(file,level); // **************************************************** - MPI_Barrier(comm); - MPI_Comm_free(&comm); } // Limit scope so variables that contain communicators will free before MPI_Finialize diff --git a/tests/lbpm_electrokinetic_SingleFluid_simulator.cpp b/tests/lbpm_electrokinetic_SingleFluid_simulator.cpp index 689745b9..b9f215e7 100644 --- a/tests/lbpm_electrokinetic_SingleFluid_simulator.cpp +++ b/tests/lbpm_electrokinetic_SingleFluid_simulator.cpp @@ -26,21 +26,22 @@ int main(int argc, char **argv) Utilities::startup( argc, argv ); { // Limit scope so variables that contain communicators will free before MPI_Finialize - - MPI_Comm comm; - MPI_Comm_dup(MPI_COMM_WORLD,&comm); - int rank = comm_rank(comm); - int nprocs = comm_size(comm); + // Initialize MPI + Utilities::startup( argc, argv ); + Utilities::MPI comm( MPI_COMM_WORLD ); + int rank = comm.getRank(); + int nprocs = comm.getSize(); if (rank == 0){ printf("********************************************************\n"); printf("Running LBPM electrokinetic single-fluid solver \n"); printf("********************************************************\n"); } - // Initialize compute device - ScaLBL_SetDevice(rank); - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); + // Initialize compute device + int device=ScaLBL_SetDevice(rank); + NULL_USE( device ); + ScaLBL_DeviceBarrier(); + comm.barrier(); PROFILE_ENABLE(1); //PROFILE_ENABLE_TRACE(); @@ -121,9 +122,6 @@ int main(int argc, char **argv) PROFILE_STOP("Main"); PROFILE_SAVE("lbpm_electrokinetic_SingleFluid_simulator",1); // **************************************************** - - MPI_Barrier(comm); - MPI_Comm_free(&comm); } // Limit scope so variables that contain communicators will free before MPI_Finialize diff --git a/tests/lbpm_greyscaleColor_simulator.cpp b/tests/lbpm_greyscaleColor_simulator.cpp index fec85c0e..2efe8c7d 100644 --- a/tests/lbpm_greyscaleColor_simulator.cpp +++ b/tests/lbpm_greyscaleColor_simulator.cpp @@ -19,27 +19,25 @@ using namespace std; int main(int argc, char **argv) { - - // Initialize MPI and error handlers + // Initialize MPI Utilities::startup( argc, argv ); + Utilities::MPI comm( MPI_COMM_WORLD ); + int rank = comm.getRank(); + int nprocs = comm.getSize(); { // Limit scope so variables that contain communicators will free before MPI_Finialize - - MPI_Comm comm; - MPI_Comm_dup(MPI_COMM_WORLD,&comm); - int rank = comm_rank(comm); - int nprocs = comm_size(comm); - + if (rank == 0){ - printf("****************************************\n"); - printf("Running Greyscale Two-Phase Calculation \n"); - printf("****************************************\n"); + printf("****************************************\n"); + printf("Running Greyscale Two-Phase Calculation \n"); + printf("****************************************\n"); } // Initialize compute device - ScaLBL_SetDevice(rank); + int device=ScaLBL_SetDevice(rank); + NULL_USE( device ); ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - + comm.barrier(); + PROFILE_ENABLE(1); //PROFILE_ENABLE_TRACE(); //PROFILE_ENABLE_MEMORY(); @@ -61,9 +59,6 @@ int main(int argc, char **argv) PROFILE_SAVE("lbpm_greyscaleColor_simulator",1); // **************************************************** - MPI_Barrier(comm); - MPI_Comm_free(&comm); - } // Limit scope so variables that contain communicators will free before MPI_Finialize Utilities::shutdown(); diff --git a/tests/lbpm_greyscale_simulator.cpp b/tests/lbpm_greyscale_simulator.cpp index df8cb3cb..e6166116 100644 --- a/tests/lbpm_greyscale_simulator.cpp +++ b/tests/lbpm_greyscale_simulator.cpp @@ -19,54 +19,49 @@ using namespace std; int main(int argc, char **argv) { + // Initialize MPI + Utilities::startup( argc, argv ); + Utilities::MPI comm( MPI_COMM_WORLD ); + int rank = comm.getRank(); + int nprocs = comm.getSize(); - // Initialize MPI and error handlers - Utilities::startup( argc, argv ); + { // Limit scope so variables that contain communicators will free before MPI_Finialize - { // Limit scope so variables that contain communicators will free before MPI_Finialize - - MPI_Comm comm; - MPI_Comm_dup(MPI_COMM_WORLD,&comm); - int rank = comm_rank(comm); - int nprocs = comm_size(comm); - - if (rank == 0){ - printf("********************************************************\n"); - printf("Running Greyscale Single Phase Permeability Calculation \n"); - printf("********************************************************\n"); - } - // Initialize compute device - ScaLBL_SetDevice(rank); - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - - PROFILE_ENABLE(1); - //PROFILE_ENABLE_TRACE(); - //PROFILE_ENABLE_MEMORY(); - PROFILE_SYNCHRONIZE(); - PROFILE_START("Main"); - Utilities::setErrorHandlers(); - - auto filename = argv[1]; - ScaLBL_GreyscaleModel Greyscale(rank,nprocs,comm); - Greyscale.ReadParams(filename); - Greyscale.SetDomain(); - Greyscale.ReadInput(); - Greyscale.Create(); // creating the model will create data structure to match the pore structure and allocate variables - Greyscale.Initialize(); // initializing the model will set initial conditions for variables - Greyscale.Run(); - Greyscale.VelocityField(); - //Greyscale.WriteDebug(); - - PROFILE_STOP("Main"); - PROFILE_SAVE("lbpm_greyscale_simulator",1); - // **************************************************** - - MPI_Barrier(comm); - MPI_Comm_free(&comm); - - } // Limit scope so variables that contain communicators will free before MPI_Finialize + if (rank == 0){ + printf("********************************************************\n"); + printf("Running Greyscale Single Phase Permeability Calculation \n"); + printf("********************************************************\n"); + } + // Initialize compute device + int device=ScaLBL_SetDevice(rank); + NULL_USE( device ); + ScaLBL_DeviceBarrier(); + comm.barrier(); + + PROFILE_ENABLE(1); + //PROFILE_ENABLE_TRACE(); + //PROFILE_ENABLE_MEMORY(); + PROFILE_SYNCHRONIZE(); + PROFILE_START("Main"); + Utilities::setErrorHandlers(); - Utilities::shutdown(); + auto filename = argv[1]; + ScaLBL_GreyscaleModel Greyscale(rank,nprocs,comm); + Greyscale.ReadParams(filename); + Greyscale.SetDomain(); + Greyscale.ReadInput(); + Greyscale.Create(); // creating the model will create data structure to match the pore structure and allocate variables + Greyscale.Initialize(); // initializing the model will set initial conditions for variables + Greyscale.Run(); + Greyscale.VelocityField(); + //Greyscale.WriteDebug(); + + PROFILE_STOP("Main"); + PROFILE_SAVE("lbpm_greyscale_simulator",1); + // **************************************************** + + } // Limit scope so variables that contain communicators will free before MPI_Finialize + + Utilities::shutdown(); } diff --git a/tests/lbpm_minkowski_scalar.cpp b/tests/lbpm_minkowski_scalar.cpp index 22893e38..5ee1a91b 100644 --- a/tests/lbpm_minkowski_scalar.cpp +++ b/tests/lbpm_minkowski_scalar.cpp @@ -14,7 +14,7 @@ #include "common/Array.h" #include "common/Domain.h" #include "common/Communication.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "IO/MeshDatabase.h" #include "IO/Mesh.h" #include "IO/Writer.h"