// Sequential blob analysis // Reads parallel simulation data and performs connectivity analysis // and averaging on a blob-by-blob basis // James E. McClure 2014 #include #include #include "common/pmmc.h" #include "common/Communication.h" #include "analysis/analysis.h" #include "IO/MeshDatabase.h" #include "IO/Reader.h" #include "IO/Writer.h" #include "ProfilerApp.h" // Get a random number in [0 1] inline double rand2() { return static_cast(rand())/static_cast(RAND_MAX); } // Test if all ranks agree on a value bool allAgree( int x, MPI_Comm comm ) { int x2 = x; MPI_Bcast(&x2,1,MPI_INT,0,comm); int diff = x==x2 ? 0:1; int diff2 = 0; MPI_Allreduce(&diff,&diff2,1,MPI_INT,MPI_SUM,comm); return diff2==0; } template bool allAgree( const std::vector& x, MPI_Comm comm ) { std::vector x2 = x; MPI_Bcast(&x2[0],x.size()*sizeof(T)/sizeof(int),MPI_INT,0,comm); int diff = x==x2 ? 0:1; int diff2 = 0; MPI_Allreduce(&diff,&diff2,1,MPI_INT,MPI_SUM,comm); return diff2==0; } // Structure to hold a bubble struct bubble_struct { Point center; double radius; // Get the distance to the bubble inline double dist( const Point& p, double Lx, double Ly, double Lz ) const { double x = fabs(p.x-center.x); double y = fabs(p.y-center.y); double z = fabs(p.z-center.z); x = std::min(x,fabs(Lx-x)); y = std::min(y,fabs(Ly-y)); z = std::min(z,fabs(Lz-z)); return sqrt(x*x+y*y+z*z)-radius; } // Check if this bubble overlaps with rhs bool overlap( const bubble_struct& rhs, double Lx, double Ly, double Lz ) const { return dist(rhs.center,Lx,Ly,Lz) <= radius+rhs.radius; } // Create a random bubble static bubble_struct rand( double Lx, double Ly, double Lz, double R0 ) { bubble_struct bubble; bubble.center.x = Lx*rand2(); bubble.center.y = Ly*rand2(); bubble.center.z = Lz*rand2(); bubble.radius = R0; bubble.radius *= 1 + 0.4*(1.0-2.0*rand2()); return bubble; } }; // Create a random set of bubles std::vector create_bubbles( int N_bubbles, double Lx, double Ly, double Lz, MPI_Comm comm ) { int rank = comm_rank(comm); std::vector bubbles(N_bubbles); if ( rank == 0 ) { double R0 = 0.2*Lx*Ly*Lz/pow((double)N_bubbles,0.333); for (int i=0; i& bubbles, DoubleArray& Phase, DoubleArray& SignDist, double Lx, double Ly, double Lz, const RankInfoStruct rank_info ) { PROFILE_START("fillBubbleData"); int nx = Phase.size(0)-2; int ny = Phase.size(1)-2; int nz = Phase.size(2)-2; Phase.fill(1); SignDist.fill(0); for (int k=0; k tmp1(nx,ny,nz), tmp2(ngx,ngy,ngz), tmp3(ngx,ngy,ngz); fillHalo fillData1(comm,rank_info,nx,ny,nz,1,1,1,0,1); fillHalo fillData2(comm,rank_info,nx,ny,nz,abs(sx),abs(sy),abs(sz),0,1); fillData1.copy(data,tmp1); fillData2.copy(tmp1,tmp2); fillData2.fill(tmp2); for (int k=abs(sz); k factors = Utilities::factor(nprocs); int nproc[3]={1,1,1}; for (size_t i=0; i bubbles = create_bubbles(20,Lx,Ly,Lz,comm); fillBubbleData( bubbles, Phase, SignDist, Lx, Ly, Lz, rank_info ); // Communication the halos fillHalo fillData(comm,rank_info,nx,ny,nz,1,1,1,0,1); fillData.fill(Phase); fillData.fill(SignDist); // Find blob domains if ( rank==0 ) { printf("Finding blob domains\n"); } double vF=0.0; double vS=0.0; IntArray GlobalBlobID; int nblobs = ComputeGlobalBlobIDs(nx,ny,nz,rank_info, Phase,SignDist,vF,vS,GlobalBlobID,comm); if ( rank==0 ) { printf("Identified %i blobs\n",nblobs); } // Create the MeshDataStruct std::vector meshData(1); meshData[0].meshName = "domain"; meshData[0].mesh = std::shared_ptr( new IO::DomainMesh(rank_info,nx,ny,nz,Lx,Ly,Lz) ); std::shared_ptr PhaseVar( new IO::Variable() ); std::shared_ptr SignDistVar( new IO::Variable() ); std::shared_ptr BlobIDVar( new IO::Variable() ); PhaseVar->name = "phase"; PhaseVar->type = IO::VolumeVariable; PhaseVar->dim = 1; PhaseVar->data.resize(nx,ny,nz); meshData[0].vars.push_back(PhaseVar); SignDistVar->name = "SignDist"; SignDistVar->type = IO::VolumeVariable; SignDistVar->dim = 1; SignDistVar->data.resize(nx,ny,nz); meshData[0].vars.push_back(SignDistVar); BlobIDVar->name = "BlobID"; BlobIDVar->type = IO::VolumeVariable; BlobIDVar->dim = 1; BlobIDVar->data.resize(nx,ny,nz); meshData[0].vars.push_back(BlobIDVar); // Save the results fillData.copy(Phase,PhaseVar->data); fillData.copy(SignDist,SignDistVar->data); fillData.copy(GlobalBlobID,BlobIDVar->data); IO::writeData( 0, meshData, 2, comm ); writeIDMap(ID_map_struct(nblobs),0,"lbpm_id_map.txt"); int save_it = 1; // Check the results int N_errors = 0; if ( nblobs != (int) bubbles.size() ) { printf("Error, detected number of bubbles %i does not match expected %i\n",nblobs,(int)bubbles.size()); N_errors++; } // Move the blobs and connect them to the previous ids PROFILE_START("constant velocity test"); if ( rank==0 ) { printf("Running constant velocity blob test\n"); } int id_max = nblobs-1; for (int i=0; i<20; i++, save_it++) { // Shift all the data shift_data( Phase, 3, -2, 1, rank_info, comm ); shift_data( SignDist, 3, -2, 1, rank_info, comm ); // Find blob domains IntArray GlobalBlobID2; int nblobs2 = ComputeGlobalBlobIDs(nx,ny,nz,rank_info, Phase,SignDist,vF,vS,GlobalBlobID2,comm); if ( nblobs2 != nblobs ) { printf("Error, number of blobs changed under constant velocity (%i,%i)\n",nblobs,nblobs2); N_errors++; } // Identify the blob maps and renumber the ids ID_map_struct map = computeIDMap(nx,ny,nz,GlobalBlobID,GlobalBlobID2,comm); std::swap(GlobalBlobID,GlobalBlobID2); std::vector new_list; getNewIDs(map,id_max,new_list); renumberIDs(new_list,GlobalBlobID); writeIDMap(map,save_it,"lbpm_id_map.txt"); bool pass = (int)map.src_dst.size()==nblobs; pass = pass && map.created.empty(); pass = pass && map.destroyed.empty(); for (size_t j=0; jdata); fillData.copy(SignDist,SignDistVar->data); fillData.copy(GlobalBlobID,BlobIDVar->data); IO::writeData( save_it, meshData, 2, comm ); } PROFILE_STOP("constant velocity test"); // Move the bubbles in independent directions to test merge/splitting of bubbles PROFILE_START("moving bubble test"); if ( rank==0 ) { printf("Running moving bubble test\n"); } std::vector velocity(bubbles.size()); if ( rank==0 ) { for (size_t i=0; idata); fillData.copy(SignDist,SignDistVar->data); fillData.copy(GlobalBlobID,BlobIDVar->data); IO::writeData( save_it, meshData, 2, comm ); save_it++; id_max = nblobs-1; for (int i=0; i<25; i++, save_it++) { // Move the bubbles for (size_t j=0; j new_list; getNewIDs(map,id_max,new_list); // Check id_max if ( !allAgree(id_max,comm) ) { if ( rank==0 ) printf("All ranks do not agree on id_max\n"); N_errors++; break; } // Check that the new id list matches on all ranks if ( !allAgree(new_list,comm) ) { if ( rank==0 ) printf("All ranks do not agree on new_list\n"); N_errors++; break; } // Renumber the ids and write the map renumberIDs(new_list,GlobalBlobID); writeIDMap(map,save_it,"lbpm_id_map.txt"); // Check the number of blobs in the map int N1 = 0; int N2 = 0; if ( rank==0 ) { if ( !map.destroyed.empty() ) { N1 += map.destroyed.size(); printf(" %i: destroyed:",i+1); for (size_t j=0; jdata); fillData.copy(SignDist,SignDistVar->data); fillData.copy(GlobalBlobID,BlobIDVar->data); IO::writeData( save_it, meshData, 2, comm ); } PROFILE_STOP("moving bubble test"); // Finished PROFILE_STOP("main"); PROFILE_SAVE("TestBlobIdentify",false); MPI_Barrier(comm); MPI_Finalize(); return N_errors; }