OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactoring interface to analysis routines

Browse Source
This commit is contained in:
James E McClure
2018-01-26 09:29:48 -05:00
parent 866499f585
commit fd17cb35f1
4 changed files with 45 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
common/Domain.h
View File

@@ -500,25 +500,20 @@ inline void WriteLocalSolidDistance(char *FILENAME, double *Distance, int N)
}
inline void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cDistEven, const double *cDistOdd, int N)
inline void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, int Np)
{
int q,n;
double value;
ofstream File(FILENAME,ios::binary);
for (n=0; n<N; n++){
for (n=0; n<Np; n++){
// Write the two density values
value = cDen[n];
File.write((char*) &value, sizeof(value));
value = cDen[N+n];
value = cDen[Np+n];
File.write((char*) &value, sizeof(value));
// Write the even distributions
for (q=0; q<10; q++){
value = cDistEven[q*N+n];
File.write((char*) &value, sizeof(value));
}
// Write the odd distributions
for (q=0; q<9; q++){
value = cDistOdd[q*N+n];
for (q=0; q<19; q++){
value = cfq[q*Np+n];
File.write((char*) &value, sizeof(value));
}
}
@@ -526,30 +521,21 @@ inline void WriteCheckpoint(const char *FILENAME, const double *cDen, const doub
}
inline void ReadCheckpoint(char *FILENAME, double *cDen, double *cDistEven, double *cDistOdd, int N)
inline void ReadCheckpoint(char *FILENAME, double *cDen, double *cfq, int Np)
{
int q=0, n=0;
double value=0;
ifstream File(FILENAME,ios::binary);
for (n=0; n<N; n++){
for (n=0; n<Np; n++){
// Write the two density values
File.read((char*) &value, sizeof(value));
cDen[n] = value;
// if (n== 66276) printf("Density a = %f \n",value);
File.read((char*) &value, sizeof(value));
cDen[N+n] = value;
// if (n== 66276) printf("Density b = %f \n",value);
cDen[Np+n] = value;
// Read the even distributions
for (q=0; q<10; q++){
for (q=0; q<19; q++){
File.read((char*) &value, sizeof(value));
cDistEven[q*N+n] = value;
// if (n== 66276) printf("dist even %i = %f \n",q,value);
}
// Read the odd distributions
for (q=0; q<9; q++){
File.read((char*) &value, sizeof(value));
cDistOdd[q*N+n] = value;
// if (n== 66276) printf("dist even %i = %f \n",q,value);
cfq[q*Np+n] = value;
}
}
File.close();

9
common/ScaLBL.h
View File

@@ -4269,23 +4269,30 @@ void ScaLBL_Communicator::RecvHalo(double *data){
}
void ScaLBL_Communicator::RegularLayout(IntArray map, double *data, double *regdata){
// Gets data from the device and stores in regular layout
int i,j,k,n,idx;
int Nx = map.size(0);
int Ny = map.size(1);
int Nz = map.size(2);
double *TmpDat;
TmpDat = new double [Np];
ScaLBL_CopyToHost(&TmpDat[0],&data[0], Np*sizeof(double));
for (k=0; k<Nz; k++){
for (j=0; j<Ny; j++){
for (i=0; i<Nx; i++){
n=k*Nx*Ny+j*Nx+i;
idx=map(i,j,k);
if (!(idx<0)){
double value=data[idx];
double value=TmpDat[idx];
regdata[n]=value;
}
}
}
}
delete [] TmpDat;
}

30
tests/lbpm_color_macro_simulator.cpp
View File

@@ -471,18 +471,15 @@ int main(int argc, char **argv)
MPI_Bcast(&timestep,1,MPI_INT,0,comm);
// Read in the restart file to CPU buffers
double *cDen = new double[2*N];
double *cDistEven = new double[10*N];
double *cDistOdd = new double[9*N];
ReadCheckpoint(LocalRestartFile, cDen, cDistEven, cDistOdd, N);
double *cDen = new double[2*Np];
double *cfq = new double[19*Np];
ReadCheckpoint(LocalRestartFile, cDen, cfq, Np);
// Copy the restart data to the GPU
ScaLBL_CopyToDevice(f_even,cDistEven,10*N*sizeof(double));
ScaLBL_CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double));
ScaLBL_CopyToDevice(Den,cDen,2*N*sizeof(double));
ScaLBL_CopyToDevice(fq,cfq,19*Np*sizeof(double));
ScaLBL_CopyToDevice(Den,cDen,2*Np*sizeof(double));
ScaLBL_DeviceBarrier();
delete [] cDen;
delete [] cDistEven;
delete [] cDistOdd;
delete [] cfq;
MPI_Barrier(comm);
}
@@ -508,15 +505,10 @@ int main(int argc, char **argv)
//...........................................................................
ScaLBL_DeviceBarrier();
ScaLBL_CopyToHost(Averages->Phase.data(),Phi,N*sizeof(double));
ScaLBL_CopyToHost(Averages->Press.data(),Pressure,Np*sizeof(double));
ScaLBL_CopyToHost(Averages->Vel_x.data(),&Velocity[0],Np*sizeof(double));
ScaLBL_CopyToHost(Averages->Vel_y.data(),&Velocity[N],Np*sizeof(double));
ScaLBL_CopyToHost(Averages->Vel_z.data(),&Velocity[2*N],Np*sizeof(double));
double *TmpDat;
TmpDat = new double [Np];
ScaLBL_CopyToHost(&TmpDat[0],&Pressure[0], Np*sizeof(double));
ScaLBL_Comm.RegularLayout(Map,TmpDat,Averages->Press.data());
ScaLBL_Comm.RegularLayout(Map,Pressure,Averages->Press.data());
ScaLBL_Comm.RegularLayout(Map,&Velocity[0],Averages->Vel_x.data());
ScaLBL_Comm.RegularLayout(Map,&Velocity[Np],Averages->Vel_y.data());
ScaLBL_Comm.RegularLayout(Map,&Velocity[2*Np],Averages->Vel_z.data());
//...........................................................................
if (rank==0) printf("********************************************************\n");
@@ -660,7 +652,7 @@ int main(int argc, char **argv)
// Run the analysis
run_analysis(timestep,RESTART_INTERVAL,rank_info,*Averages,last_ids,last_index,last_id_map,
Nx,Ny,Nz,pBC,beta,err,Phi,Pressure,Velocity,ID,f_even,f_odd,Den,
Np,Nx,Ny,Nz,pBC,beta,err,Phi,Pressure,Velocity,ID,fq,Den,
LocalRestartFile,meshData,fillData,tpool,work_ids);
// Save the timers

36
tests/lbpm_color_simulator.h
View File

@@ -33,18 +33,18 @@ class WriteRestartWorkItem: public ThreadPool::WorkItem
{
public:
WriteRestartWorkItem( const char* filename_, std::shared_ptr<double> cDen_,
std::shared_ptr<double> cDistEven_, std::shared_ptr<double>cDistOdd_, int N_ ):
filename(filename_), cDen(cDen_), cDistEven(cDistEven_), cDistOdd(cDistOdd_), N(N_) {}
std::shared_ptr<double> cfq_, int N_ ):
filename(filename_), cDen(cDen_), cfq(cfq_), N(N_) {}
virtual void run() {
PROFILE_START("Save Checkpoint",1);
WriteCheckpoint(filename,cDen.get(),cDistEven.get(),cDistOdd.get(),N);
WriteCheckpoint(filename,cDen.get(),cfq.get(),N);
PROFILE_STOP("Save Checkpoint",1);
};
virtual bool has_result() const { return false; }
private:
WriteRestartWorkItem();
const char* filename;
std::shared_ptr<double> cDen, cDistEven, cDistOdd;
std::shared_ptr<double> cDen, cfq;
const int N;
};
@@ -222,15 +222,13 @@ private:
double beta;
};
// Function to start the analysis
void run_analysis( int timestep, int restart_interval,
const RankInfoStruct& rank_info, TwoPhase& Averages,
BlobIDstruct& last_ids, BlobIDstruct& last_index, BlobIDList& last_id_map,
int Nx, int Ny, int Nz, bool pBC, double beta, double err,
int Np, int Nx, int Ny, int Nz, bool pBC, double beta, double err,
const double *Phi, double *Pressure, const double *Velocity,
const char *ID, const double *f_even, const double *f_odd, const double *Den,
const char *ID, const double *fq, const double *Den,
const char *LocalRestartFile, std::vector<IO::MeshDataStruct>& visData, fillHalo<double>& fillData,
ThreadPool& tpool, AnalysisWaitIdStruct& wait )
{
@@ -312,21 +310,19 @@ void run_analysis( int timestep, int restart_interval,
PROFILE_STOP("Copy-Wait",1);
PROFILE_START("Copy-State",1);
memcpy(Averages.Phase.data(),phase->data(),N*sizeof(double));
ScaLBL_CopyToHost(Averages.Press.data(),Pressure,N*sizeof(double));
ScaLBL_CopyToHost(Averages.Vel_x.data(),&Velocity[0],N*sizeof(double));
ScaLBL_CopyToHost(Averages.Vel_y.data(),&Velocity[N],N*sizeof(double));
ScaLBL_CopyToHost(Averages.Vel_z.data(),&Velocity[2*N],N*sizeof(double));
ScaLBL_Comm.RegularLayout(Map,Pressure,Averages->Press.data());
ScaLBL_Comm.RegularLayout(Map,&Velocity[0],Averages->Vel_x.data());
ScaLBL_Comm.RegularLayout(Map,&Velocity[Np],Averages->Vel_y.data());
ScaLBL_Comm.RegularLayout(Map,&Velocity[2*Np],Averages->Vel_z.data());
PROFILE_STOP("Copy-State",1);
}
std::shared_ptr<double> cDen, cDistEven, cDistOdd;
std::shared_ptr<double> cDen, cfq;
if ( (type&CreateRestart) != 0 ) {
// Copy restart data to the CPU
cDen = std::shared_ptr<double>(new double[2*N],DeleteArray<double>);
cDistEven = std::shared_ptr<double>(new double[10*N],DeleteArray<double>);
cDistOdd = std::shared_ptr<double>(new double[9*N],DeleteArray<double>);
ScaLBL_CopyToHost(cDistEven.get(),f_even,10*N*sizeof(double));
ScaLBL_CopyToHost(cDistOdd.get(),f_odd,9*N*sizeof(double));
ScaLBL_CopyToHost(cDen.get(),Den,2*N*sizeof(double));
cDen = std::shared_ptr<double>(new double[2*Np],DeleteArray<double>);
cfq = std::shared_ptr<double>(new double[19*Np],DeleteArray<double>);
ScaLBL_CopyToHost(cfq.get(),fq,19*Np*sizeof(double));
ScaLBL_CopyToHost(cDen.get(),Den,2*Np*sizeof(double));
}
PROFILE_STOP("Copy data to host",1);
@@ -376,7 +372,7 @@ void run_analysis( int timestep, int restart_interval,
fclose(Rst);
}
// Write the restart file (using a seperate thread)
WriteRestartWorkItem *work = new WriteRestartWorkItem(LocalRestartFile,cDen,cDistEven,cDistOdd,N);
WriteRestartWorkItem *work = new WriteRestartWorkItem(LocalRestartFile,cDen,cfq,Np);
work->add_dependency(wait.restart);
wait.restart = tpool.add_work(work);
}